summaryrefslogtreecommitdiff
path: root/src/cuda-sim
diff options
context:
space:
mode:
authorMahmoud <[email protected]>2019-07-29 21:18:06 -0400
committerMahmoud <[email protected]>2019-07-29 21:18:06 -0400
commit5875fda72d4402413bc5c04ee5ec15085ff2b90a (patch)
treec920268d899df331e1a5e451a35133eaff7ca341 /src/cuda-sim
parentc05dc90da35fc2bd9dd42da9626bf3a60e2c9e8d (diff)
parent21d937256fbca004c926531cfef1adefcedeef91 (diff)
Merge branch 'dev' of https://github.com/mkhairy/gpgpu-sim-private into dev-purdue-integration-trace
Diffstat (limited to 'src/cuda-sim')
-rw-r--r--src/cuda-sim/cuda-math.h4
-rw-r--r--src/cuda-sim/cuda-sim.cc596
-rw-r--r--src/cuda-sim/cuda-sim.h7
-rw-r--r--src/cuda-sim/cuda_device_runtime.cc17
-rw-r--r--src/cuda-sim/half.h3067
-rw-r--r--src/cuda-sim/half.hpp3067
-rw-r--r--src/cuda-sim/instructions.cc1385
-rw-r--r--src/cuda-sim/memory.cc12
-rw-r--r--src/cuda-sim/memory.h5
-rw-r--r--src/cuda-sim/opcodes.def5
-rw-r--r--src/cuda-sim/opcodes.h13
-rw-r--r--src/cuda-sim/ptx.l45
-rw-r--r--src/cuda-sim/ptx.y36
-rw-r--r--src/cuda-sim/ptx_ir.cc110
-rw-r--r--src/cuda-sim/ptx_ir.h140
-rw-r--r--src/cuda-sim/ptx_loader.cc238
-rw-r--r--src/cuda-sim/ptx_loader.h3
-rw-r--r--src/cuda-sim/ptx_parser.cc165
-rw-r--r--src/cuda-sim/ptx_parser.h2
-rw-r--r--src/cuda-sim/ptx_sim.cc17
-rw-r--r--src/cuda-sim/ptx_sim.h35
21 files changed, 8669 insertions, 300 deletions
diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h
index f88c526..a5db337 100644
--- a/src/cuda-sim/cuda-math.h
+++ b/src/cuda-sim/cuda-math.h
@@ -64,6 +64,7 @@
* the above Disclaimer and U.S. Government End Users Notice.
*/
+
#ifndef CUDA_MATH
#define CUDA_MATH
@@ -150,6 +151,7 @@ float __ll2float_rd(long long int a) {
#include <device_types.h>
#include <fenv.h>
+
// 32-bit integer to float
float __int2float_rn(int a) {
int orig_rnd_mode = fegetround();
@@ -359,7 +361,7 @@ int __signbitd(double d)
#ifdef __APPLE__
int isnanf(float a)
{
- return (isnan(a));
+ return (std::isnan(a));
}
#endif
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 9246613..4c0fc58 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -33,7 +33,7 @@
#include "ptx.tab.h"
#include "ptx_sim.h"
#include <stdio.h>
-
+#include <sstream>
#include "opcodes.h"
#include "../statwrapper.h"
#include <set>
@@ -58,12 +58,14 @@ int g_debug_execution = 0;
int g_debug_thread_uid = 0;
addr_t g_debug_pc = 0xBEEF1518;
// Output debug information to file options
+int cp_count;
+int cp_cta_resume;
unsigned g_ptx_sim_num_insn = 0;
unsigned gpgpu_param_num_shaders = 0;
-char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu;
-char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu;
+char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor;
+char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu,*opcode_initiation_tensor;
char *cdp_latency_str;
unsigned cdp_latency[5];
@@ -84,6 +86,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) {
"Opcode latencies for SFU instructions"
"Default 8",
"8");
+ option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor,
+ "Opcode latencies for Tensor instructions"
+ "Default 64",
+ "64");
option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int,
"Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>"
"Default 1,1,4,4,32",
@@ -100,6 +106,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) {
"Opcode initiation intervals for sfu instructions"
"Default 8",
"8");
+ option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor,
+ "Opcode initiation intervals for tensor instructions"
+ "Default 64",
+ "64");
option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str,
"CDP API latency <cudaStreamCreateWithFlags, \
cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \
@@ -113,22 +123,35 @@ static address_type get_converge_point(address_type pc);
void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext)
{
std::string texname(name);
- m_NameToTextureRef[texname] = texref;
+ if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){
+ m_NameToTextureRef[texname] = std::set<const struct textureReference*>();
+ }else{
+ const struct textureReference* tr = *m_NameToTextureRef[texname].begin();
+ assert(tr!=NULL);
+ //asserts that all texrefs in set have same fields
+ assert(tr->normalized==texref->normalized&&
+ tr->filterMode==texref->filterMode&&
+ tr->addressMode[0]==texref->addressMode[0]&&
+ tr->addressMode[1]==texref->addressMode[1]&&
+ tr->addressMode[2]==texref->addressMode[2]&&
+ tr->channelDesc.x==texref->channelDesc.x&&
+ tr->channelDesc.y==texref->channelDesc.y&&
+ tr->channelDesc.z==texref->channelDesc.z&&
+ tr->channelDesc.w==texref->channelDesc.w&&
+ tr->channelDesc.f==texref->channelDesc.f
+ );
+ }
+ m_NameToTextureRef[texname].insert(texref);
+ m_TextureRefToName[texref] = texname;
const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext);
- m_TextureRefToAttribute[texref] = texAttr;
+ m_NameToAttribute[texname] = texAttr;
}
const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref)
{
- std::map<std::string, const struct textureReference*>::iterator itr = m_NameToTextureRef.begin();
- while (itr != m_NameToTextureRef.end()) {
- if ((*itr).second == texref) {
- const char *p = ((*itr).first).c_str();
- return p;
- }
- itr++;
- }
- return NULL;
+ std::map<const struct textureReference*, std::string>::const_iterator t=m_TextureRefToName.find(texref);
+ assert( t != m_TextureRefToName.end() );
+ return t->second.c_str();
}
unsigned int intLOGB2( unsigned int v ) {
@@ -148,7 +171,14 @@ unsigned int intLOGB2( unsigned int v ) {
void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array)
{
- m_TextureRefToCudaArray[texref] = array;
+ std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref);
+
+ std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname);
+ //check that there's nothing there first
+ if(t != m_NameToCudaArray.end()){
+ printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str());
+ }
+ m_NameToCudaArray[texname] = array;
unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z;
unsigned int texel_size = texel_size_bits/8;
unsigned int Tx, Ty;
@@ -188,7 +218,15 @@ void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* te
texInfo->Ty_numbits = intLOGB2(Ty);
texInfo->texel_size = texel_size;
texInfo->texel_size_numbits = intLOGB2(texel_size);
- m_TextureRefToTexureInfo[texref] = texInfo;
+ m_NameToTextureInfo[texname] = texInfo;
+}
+
+void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref)
+{
+ //assumes bind-use-unbind-bind-use-unbind pattern
+ std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref);
+ m_NameToCudaArray.erase(texname);
+ m_NameToTextureInfo.erase(texname);
}
unsigned g_assemble_code_next_pc=0;
@@ -220,7 +258,9 @@ void function_info::ptx_assemble()
m_start_PC = PC;
addr_t n=0; // offset in m_instr_mem
- s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size());
+ //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative.
+ //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size());
+ s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size());
for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) {
ptx_instruction *pI = *i;
if ( pI->is_label() ) {
@@ -258,11 +298,13 @@ void function_info::ptx_assemble()
target.set_type(label_t);
}
}
-
+ m_n = n;
printf(" done.\n");
fflush(stdout);
- printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() );
+ //disable pdom analysis here and do it at runtime
+#if 0
+ printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() );
create_basic_blocks();
connect_basic_blocks();
bool modified = false;
@@ -296,6 +338,7 @@ void function_info::ptx_assemble()
fflush(stdout);
m_assembled = true;
+#endif
}
addr_t shared_to_generic( unsigned smid, addr_t addr )
@@ -536,7 +579,7 @@ void ptx_instruction::set_mul_div_or_other_archop(){
sp_op=FP_EXP_OP;
break;
default:
- if(op==ALU_OP)
+ if((op==ALU_OP)||(op==TENSOR_CORE_OP))
sp_op=FP__OP;
break;
@@ -558,7 +601,7 @@ void ptx_instruction::set_mul_div_or_other_archop(){
sp_op=INT_DIV_OP;
break;
default:
- if(op==ALU_OP)
+ if((op==ALU_OP))
sp_op=INT__OP;
break;
}
@@ -597,6 +640,9 @@ void ptx_instruction::set_bar_type()
abort();
}
}
+ else if(m_opcode==SST_OP) {
+ bar_type = SYNC;
+ }
}
@@ -606,10 +652,12 @@ void ptx_instruction::set_opcode_and_latency()
unsigned fp_latency[5];
unsigned dp_latency[5];
unsigned sfu_latency;
+ unsigned tensor_latency;
unsigned int_init[5];
unsigned fp_init[5];
unsigned dp_init[5];
unsigned sfu_init;
+ unsigned tensor_init;
/*
* [0] ADD,SUB
* [1] MAX,Min
@@ -628,6 +676,8 @@ void ptx_instruction::set_opcode_and_latency()
&dp_latency[3],&dp_latency[4]);
sscanf(opcode_latency_sfu, "%u",
&sfu_latency);
+ sscanf(opcode_latency_tensor, "%u",
+ &tensor_latency);
sscanf(opcode_initiation_int, "%u,%u,%u,%u,%u",
&int_init[0],&int_init[1],&int_init[2],
&int_init[3],&int_init[4]);
@@ -639,6 +689,8 @@ void ptx_instruction::set_opcode_and_latency()
&dp_init[3],&dp_init[4]);
sscanf(opcode_initiation_sfu, "%u",
&sfu_init);
+ sscanf(opcode_initiation_tensor, "%u",
+ &tensor_init);
sscanf(cdp_latency_str, "%u,%u,%u,%u,%u",
&cdp_latency[0],&cdp_latency[1],&cdp_latency[2],
&cdp_latency[3],&cdp_latency[4]);
@@ -662,13 +714,16 @@ void ptx_instruction::set_opcode_and_latency()
if ( has_memory_write() ) op = STORE_OP;
break;
case LD_OP: op = LOAD_OP; break;
+ case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break;
case LDU_OP: op = LOAD_OP; break;
case ST_OP: op = STORE_OP; break;
+ case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break;
case BRA_OP: op = BRANCH_OP; break;
case BREAKADDR_OP: op = BRANCH_OP; break;
case TEX_OP: op = LOAD_OP; mem_op=TEX; break;
case ATOM_OP: op = LOAD_OP; break;
case BAR_OP: op = BARRIER_OP; break;
+ case SST_OP: op = BARRIER_OP; break;
case MEMBAR_OP: op = MEMORY_BARRIER_OP; break;
case CALL_OP:
{
@@ -695,6 +750,7 @@ void ptx_instruction::set_opcode_and_latency()
case F32_TYPE:
latency = fp_latency[0];
initiation_interval = fp_init[0];
+ op = SP_OP;
break;
case F64_TYPE:
case FF64_TYPE:
@@ -708,6 +764,7 @@ void ptx_instruction::set_opcode_and_latency()
default: //Use int settings for default
latency = int_latency[0];
initiation_interval = int_init[0];
+ op = INTP_OP;
break;
}
break;
@@ -717,6 +774,7 @@ void ptx_instruction::set_opcode_and_latency()
case F32_TYPE:
latency = fp_latency[1];
initiation_interval = fp_init[1];
+ op = SP_OP;
break;
case F64_TYPE:
case FF64_TYPE:
@@ -730,6 +788,7 @@ void ptx_instruction::set_opcode_and_latency()
default: //Use int settings for default
latency = int_latency[1];
initiation_interval = int_init[1];
+ op = INTP_OP;
break;
}
break;
@@ -739,6 +798,7 @@ void ptx_instruction::set_opcode_and_latency()
case F32_TYPE:
latency = fp_latency[2];
initiation_interval = fp_init[2];
+ op = SP_OP;
break;
case F64_TYPE:
case FF64_TYPE:
@@ -752,7 +812,7 @@ void ptx_instruction::set_opcode_and_latency()
default: //Use int settings for default
latency = int_latency[2];
initiation_interval = int_init[2];
- op = SFU_OP;
+ op = INTP_OP;
break;
}
break;
@@ -762,6 +822,7 @@ void ptx_instruction::set_opcode_and_latency()
case F32_TYPE:
latency = fp_latency[3];
initiation_interval = fp_init[3];
+ op = SP_OP;
break;
case F64_TYPE:
case FF64_TYPE:
@@ -775,7 +836,7 @@ void ptx_instruction::set_opcode_and_latency()
default: //Use int settings for default
latency = int_latency[3];
initiation_interval = int_init[3];
- op = SFU_OP;
+ op = INTP_OP;
break;
}
break;
@@ -802,13 +863,17 @@ void ptx_instruction::set_opcode_and_latency()
}
break;
case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP:
- //Using double to approximate those
latency = sfu_latency;
initiation_interval = sfu_init;
op = SFU_OP;
break;
+ case MMA_OP:
+ latency = tensor_latency;
+ initiation_interval = tensor_init;
+ op=TENSOR_CORE_OP;
+ break;
case SHFL_OP:
- latency = 32;
+ latency = 4;
initiation_interval = 4;
break;
default:
@@ -863,10 +928,14 @@ void ptx_instruction::pre_decode()
{
pc = m_PC;
isize = m_inst_size;
- for( unsigned i=0; i<4; i++) {
+ for(unsigned i=0; i<MAX_OUTPUT_VALUES; i++) {
out[i] = 0;
+ }
+ for(unsigned i=0; i<MAX_INPUT_VALUES; i++) {
in[i] = 0;
}
+ incount=0;
+ outcount=0;
is_vectorin = 0;
is_vectorout = 0;
std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1);
@@ -898,6 +967,7 @@ void ptx_instruction::pre_decode()
switch( m_cache_option ) {
case CA_OPTION: cache_op = CACHE_ALL; break;
+ case NC_OPTION: cache_op = CACHE_L1; break;
case CG_OPTION: cache_op = CACHE_GLOBAL; break;
case CS_OPTION: cache_op = CACHE_STREAMING; break;
case LU_OPTION: cache_op = CACHE_LAST_USE; break;
@@ -905,9 +975,11 @@ void ptx_instruction::pre_decode()
case WB_OPTION: cache_op = CACHE_WRITE_BACK; break;
case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break;
default:
- if( m_opcode == LD_OP || m_opcode == LDU_OP )
+ //if( m_opcode == LD_OP || m_opcode == LDU_OP )
+ if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP )
cache_op = CACHE_ALL;
- else if( m_opcode == ST_OP )
+ //else if( m_opcode == ST_OP )
+ else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP )
cache_op = CACHE_WRITE_BACK;
else if( m_opcode == ATOM_OP )
cache_op = CACHE_GLOBAL;
@@ -933,6 +1005,10 @@ void ptx_instruction::pre_decode()
if( num_elem >= 2 ) out[1] = o.reg2_num();
if( num_elem >= 3 ) out[2] = o.reg3_num();
if( num_elem >= 4 ) out[3] = o.reg4_num();
+ if( num_elem >= 5 ) out[4] = o.reg5_num();
+ if( num_elem >= 6 ) out[5] = o.reg6_num();
+ if( num_elem >= 7 ) out[6] = o.reg7_num();
+ if( num_elem >= 8 ) out[7] = o.reg8_num();
for (int i = 0; i < num_elem; i++)
arch_reg.dst[i] = o.arch_reg_num(i);
}
@@ -951,16 +1027,29 @@ void ptx_instruction::pre_decode()
//assert(m == 0); //only support 1 vector operand (for textures) right now
is_vectorout = 1;
unsigned num_elem = o.get_vect_nelem();
- if( num_elem >= 1 ) in[0] = o.reg1_num();
- if( num_elem >= 2 ) in[1] = o.reg2_num();
- if( num_elem >= 3 ) in[2] = o.reg3_num();
- if( num_elem >= 4 ) in[3] = o.reg4_num();
+ if( num_elem >= 1 ) in[m+0] = o.reg1_num();
+ if( num_elem >= 2 ) in[m+1] = o.reg2_num();
+ if( num_elem >= 3 ) in[m+2] = o.reg3_num();
+ if( num_elem >= 4 ) in[m+3] = o.reg4_num();
+ if( num_elem >= 5 ) in[m+4] = o.reg5_num();
+ if( num_elem >= 6 ) in[m+5] = o.reg6_num();
+ if( num_elem >= 7 ) in[m+6] = o.reg7_num();
+ if( num_elem >= 8 ) in[m+7] = o.reg8_num();
for (int i = 0; i < num_elem; i++)
- arch_reg.src[i] = o.arch_reg_num(i);
- m+=4;
+ arch_reg.src[m+i] = o.arch_reg_num(i);
+ m+=num_elem;
}
}
}
+
+ //Setting number of input and output operands which is required for scoreboard check
+ for(int i=0;i<MAX_OUTPUT_VALUES;i++)
+ if(out[i]>0)
+ outcount++;
+
+ for(int i=0;i<MAX_INPUT_VALUES;i++)
+ if(in[i]>0)
+ incount++;
// Get predicate
if(has_pred()) {
@@ -1158,14 +1247,19 @@ void function_info::finalize( memory_space *param_mem )
}
// copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over
//Jin: copy parameter using aligned rules
+ const type_info *paramtype = param->type();
+ int align_amount = paramtype->get_key().get_alignment_spec();
+ align_amount = (align_amount == -1) ? size : align_amount;
+ param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned
+
const size_t word_size = 4;
- //param_address = (param_address + size - 1) / size * size; //aligned with size TODO: align not correct
+ //param_address = (param_address + size - 1) / size * size; //aligned with size
for (size_t idx = 0; idx < size; idx += word_size) {
const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic
param_mem->write(param_address + idx, word_size, pdata,NULL,NULL);
}
unsigned offset = p.get_offset();
- //assert(offset == param_address);
+ assert(offset == param_address);
param->set_address(param_address);
param_address += size;
}
@@ -1212,6 +1306,165 @@ void function_info::list_param( FILE *fout ) const
fflush(fout);
}
+void function_info::ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim)
+{
+ static unsigned long long counter = 0;
+ std::vector< std::pair<size_t, unsigned char*> > param_data;
+ std::vector<unsigned> offsets;
+ std::vector<bool> paramIsPointer;
+
+ char * gpgpusim_path = getenv("GPGPUSIM_ROOT");
+ assert(gpgpusim_path!=NULL);
+ char * wys_exec_path = getenv("WYS_EXEC_PATH");
+ assert(wys_exec_path!=NULL);
+ std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data";
+ std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter));
+
+ //initialize paramList
+ char buff[1024];
+ std::string filename_c(filename+"_c");
+ snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str());
+ system(buff);
+ FILE *fp = fopen(filename_c.c_str(), "r");
+ fgets(buff, 1024, fp);
+ fclose(fp);
+ std::string fn(buff);
+ size_t pos1, pos2;
+ pos1 = fn.find_last_of("(");
+ pos2 = fn.find(")", pos1);
+ assert(pos2>pos1&&pos1>0);
+ strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str());
+ char *tok;
+ tok = strtok(buff, ",");
+ std::string tmp;
+ while(tok!=NULL){
+ std::string param(tok);
+ if(param.find("<")!=std::string::npos){
+ assert(param.find(">")==std::string::npos);
+ assert(param.find("*")==std::string::npos);
+ tmp = param;
+ } else {
+ if (tmp.length()>0){
+ tmp = "";
+ assert(param.find(">")!=std::string::npos);
+ assert(param.find("<")==std::string::npos);
+ assert(param.find("*")==std::string::npos);
+ }
+ printf("%s\n", param.c_str());
+ if(param.find("*")!=std::string::npos){
+ paramIsPointer.push_back(true);
+ }else{
+ paramIsPointer.push_back(false);
+ }
+ }
+ tok = strtok(NULL, ",");
+ }
+
+
+ for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) {
+ param_info &p = i->second;
+ std::string name = p.get_name();
+ symbol *param = m_symtab->lookup(name.c_str());
+ addr_t param_addr = param->get_address();
+ param_t param_value = p.get_value();
+ offsets.push_back((unsigned)p.get_offset());
+
+ if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){
+ //is pointer
+ assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer");
+ size_t array_size = 0;
+ unsigned long long param_pointer = *(unsigned long long*)param_value.pdata;
+ if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){
+ array_size = mallocPtr_Size[param_pointer];
+ }else{
+ for( std::map<unsigned long long, size_t>::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) {
+ if(param_pointer>j->first&&param_pointer<j->first + j->second){
+ array_size = j->first + j->second - param_pointer;
+ break;
+ }
+ }
+ assert(array_size>0&&"pointer was not previously malloc'd");
+ }
+
+ unsigned char* val = (unsigned char*) malloc(param_value.size);
+ param_mem->read(param_addr,param_value.size,(void*)val);
+ unsigned char* array_val = (unsigned char*) malloc(array_size);
+ gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val);
+ param_data.push_back(std::pair<size_t, unsigned char*>(array_size,array_val));
+ paramIsPointer.push_back(true);
+ }else{
+ unsigned char* val = (unsigned char*) malloc(param_value.size);
+ param_mem->read(param_addr,param_value.size,(void*)val);
+ param_data.push_back(std::pair<size_t, unsigned char*>(param_value.size,val));
+ paramIsPointer.push_back(false);
+ }
+ }
+
+ FILE *fout = fopen (filename.c_str(), "w");
+ printf("Writing data to %s ...\n", filename.c_str());
+ fprintf(fout, "%s\n", get_name().c_str());
+ fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z);
+ size_t index = 0;
+ for( std::vector< std::pair<size_t,unsigned char*> >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) {
+ if (paramIsPointer[index]){
+ fprintf(fout, "*");
+ }
+ fprintf(fout, "%lu :", i->first);
+ for (size_t j = 0; j<i->first; j++){
+ fprintf(fout, " %u", i->second[j]);
+ }
+ fprintf(fout, " : %u", offsets[index]);
+ free (i->second);
+ fprintf(fout, "\n");
+ index++;
+ }
+ fflush(fout);
+ fclose(fout);
+
+ //ptx config
+ std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter));
+ snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str());
+ if (system(buff)!=0){
+ printf("WARNING: Failed to execute grep to find ptx source \n");
+ printf("Problematic call: %s", buff);
+ abort();
+ }
+ FILE *fin = fopen(ptx_config_fn.c_str(), "r");
+ char ptx_source[256];
+ unsigned line_number;
+ int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number);
+ assert(numscanned == 2);
+ fclose(fin);
+ snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str());
+ if (system(buff)!=0){
+ printf("WARNING: Failed to execute grep to find ptx header \n");
+ printf("Problematic call: %s", buff);
+ abort();
+ }
+ fin = fopen(ptx_source, "r");
+ assert(fin!=NULL);
+ printf("Writing data to %s ...\n", ptx_config_fn.c_str());
+ fout = fopen(ptx_config_fn.c_str(), "a");
+ assert(fout!=NULL);
+ for (unsigned i = 0; i<line_number; i++){
+ fgets(buff, 1024, fin);
+ assert(!feof(fin));
+ }
+ fprintf(fout, "\n\n");
+ do{
+ fprintf(fout, "%s", buff);
+ fgets(buff, 1024, fin);
+ if(feof(fin)){
+ break;
+ }
+ } while(strstr(buff, "entry")==NULL);
+
+ fclose(fin);
+ fflush(fout);
+ fclose(fout);
+ counter++;
+}
+
template<int activate_level>
bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc)
{
@@ -1254,13 +1507,27 @@ static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *th
std::string texname = src1.name();
gpgpu_t *gpu = thread->get_gpu();
- const struct textureReference* texref = gpu->get_texref(texname);
- const struct textureInfo* texInfo = gpu->get_texinfo(texref);
+ /*
+ For programs with many streams, textures can be bound and unbound
+ asynchronously. This means we need to use the kernel's "snapshot" of
+ the state of the texture mappings when it was launched (so that we
+ don't try to access the incorrect texture mapping if it's been updated,
+ or that we don't access a mapping that has been unbound).
+ */
+ kernel_info_t& k = thread->get_kernel();
+ const struct textureInfo* texInfo = k.get_texinfo(texname);
unsigned data_size = texInfo->texel_size;
return data_size;
}
+int tensorcore_op(int inst_opcode){
+
+ if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP))
+ return 1;
+ else
+ return 0;
+}
void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
{
@@ -1269,6 +1536,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
addr_t pc = next_instr();
assert( pc == inst.pc ); // make sure timing model and functional model are in sync
const ptx_instruction *pI = m_func_info->get_instruction(pc);
+
set_npc( pc + pI->inst_size() );
@@ -1301,6 +1569,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F);
}
}
+ int inst_opcode=pI->get_opcode();
if( skip ) {
inst.set_not_active(lane_id);
@@ -1312,13 +1581,25 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
*((warp_inst_t*)pJ) = inst; // copy active mask information
pI = pJ;
}
- switch ( pI->get_opcode() ) {
-#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break;
-#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break;
-#include "opcodes.def"
-#undef OP_DEF
-#undef OP_W_DEF
- default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break;
+
+ if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){
+ if(inst.active_count()!=MAX_WARP_SIZE)
+ {
+ printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active.");
+ assert(0);
+ }
+ }
+
+ //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation
+ if(!tensorcore_op(inst_opcode)||(tensorcore_op(inst_opcode))&&(lane_id==0)){
+ switch ( inst_opcode ) {
+ #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break;
+ #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break;
+ #include "opcodes.def"
+ #undef OP_DEF
+ #undef OP_W_DEF
+ default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break;
+ }
}
delete pJ;
pI = pI_saved;
@@ -1361,13 +1642,16 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
_memory_op_t insn_memory_op = no_memory_op;
unsigned insn_data_size = 0;
if ( (pI->has_memory_read() || pI->has_memory_write()) ) {
- insn_memaddr = last_eaddr();
- insn_space = last_space();
- unsigned to_type = pI->get_type();
- insn_data_size = datatype2size(to_type);
- insn_memory_op = pI->has_memory_read() ? memory_load : memory_store;
+ if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP)))
+ {
+ insn_memaddr = last_eaddr();
+ insn_space = last_space();
+ unsigned to_type = pI->get_type();
+ insn_data_size = datatype2size(to_type);
+ insn_memory_op = pI->has_memory_read() ? memory_load : memory_store;
+ }
}
-
+
if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) {
inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/);
}
@@ -1439,12 +1723,15 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
// "Return values"
if(!skip) {
- inst.space = insn_space;
- inst.set_addr(lane_id, insn_memaddr);
- inst.data_size = insn_data_size; // simpleAtomicIntrinsics
- assert( inst.memory_op == insn_memory_op );
- }
-
+ if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP)))
+ {
+ inst.space = insn_space;
+ inst.set_addr(lane_id, insn_memaddr);
+ inst.data_size = insn_data_size; // simpleAtomicIntrinsics
+ assert( inst.memory_op == insn_memory_op );
+ }
+ }
+
} catch ( int x ) {
printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() );
printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() );
@@ -1483,6 +1770,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
std::list<ptx_thread_info *> &active_threads = kernel.active_threads();
static std::map<unsigned,memory_space*> shared_memory_lookup;
+ static std::map<unsigned,memory_space*> sstarr_memory_lookup;
static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup;
static std::map<unsigned,ptx_warp_info*> ptx_warp_lookup;
static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup;
@@ -1527,6 +1815,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
//initializing new CTA
ptx_cta_info *cta_info = NULL;
memory_space *shared_mem = NULL;
+ memory_space *sstarr_mem = NULL;
unsigned cta_size = kernel.threads_per_cta();
unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5
@@ -1544,6 +1833,9 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
snprintf(buf,512,"shared_%u", sid);
shared_mem = new memory_space_impl<16*1024>(buf,4);
shared_memory_lookup[sm_idx] = shared_mem;
+ snprintf(buf,512,"sstarr_%u", sid);
+ sstarr_mem = new memory_space_impl<16*1024>(buf,4);
+ sstarr_memory_lookup[sm_idx] = sstarr_mem;
cta_info = new ptx_cta_info(sm_idx);
ptx_cta_lookup[sm_idx] = cta_info;
} else {
@@ -1552,6 +1844,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
sm_idx, sid, max_cta_per_sm );
}
shared_mem = shared_memory_lookup[sm_idx];
+ sstarr_mem = sstarr_memory_lookup[sm_idx];
cta_info = ptx_cta_lookup[sm_idx];
cta_info->check_cta_thread_status_and_reset();
}
@@ -1564,7 +1857,6 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
kernel.increment_thread_id();
new_tid += tid;
ptx_thread_info *thd = new ptx_thread_info(kernel);
-
ptx_warp_info *warp_info = NULL;
if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) {
warp_info = new ptx_warp_info();
@@ -1593,9 +1885,11 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
thd->cpy_tid_to_reg(tid3d);
thd->set_valid();
thd->m_shared_mem = shared_mem;
+ thd->m_sstarr_mem = sstarr_mem;
function_info *finfo = thd->func_info();
symbol_table *st = finfo->get_symtab();
thd->func_info()->param_to_shared(thd->m_shared_mem,st);
+ thd->func_info()->param_to_shared(thd->m_sstarr_mem,st);
thd->m_cta_info = cta_info;
cta_info->add_thread(thd);
thd->m_local_mem = local_mem;
@@ -1632,7 +1926,7 @@ kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
struct dim3 blockDim,
gpgpu_t *gpu )
{
- kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry);
+ kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping());
unsigned argcount=args.size();
unsigned argn=1;
for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) {
@@ -1813,6 +2107,38 @@ ptx_cta_info *g_func_cta_info = NULL;
#define MAX(a,b) (((a)>(b))?(a):(b))
+unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core)
+{
+
+ unsigned int padded_cta_size = threads_per_cta;
+ if (padded_cta_size%warp_size)
+ padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size);
+ unsigned int result_thread = n_thread_per_shader / padded_cta_size;
+
+ unsigned int result_shmem = (unsigned)-1;
+ if (kernel_info->smem > 0)
+ result_shmem = gpgpu_shmem_size / kernel_info->smem;
+ unsigned int result_regs = (unsigned)-1;
+ if (kernel_info->regs > 0)
+ result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3));
+ printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) );
+ //Limit by CTA
+ unsigned int result_cta = max_cta_per_core;
+
+ unsigned result = result_thread;
+ result = gs_min2(result, result_shmem);
+ result = gs_min2(result, result_regs);
+ result = gs_min2(result, result_cta);
+
+ printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result);
+ if (result == result_thread) printf (" threads");
+ if (result == result_shmem) printf (" shmem");
+ if (result == result_regs) printf (" regs");
+ if (result == result_cta) printf (" cta_limit");
+ printf ("\n");
+
+ return result;
+}
/*!
This function simulates the CUDA code functionally, it takes a kernel_info_t parameter
which holds the data for the CUDA kernel to be executed
@@ -1823,20 +2149,71 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL )
//using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here
extern gpgpu_sim *g_the_gpu;
+ //before we execute, we should do PDOM analysis for functional simulation scenario.
+ function_info *kernel_func_info = kernel.entry();
+ const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info);
+ checkpoint *g_checkpoint;
+ g_checkpoint = new checkpoint();
+
+ if (kernel_func_info->is_pdom_set()) {
+ printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() );
+ } else {
+ printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() );
+ kernel_func_info->do_pdom();
+ kernel_func_info->set_pdom();
+ }
+
+ unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), g_the_gpu->getShaderCoreConfig()->warp_size, g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, g_the_gpu->getShaderCoreConfig()->max_cta_per_core);
+ printf("Max CTA : %d\n",max_cta_tot);
+
+
+
+
+
+ int inst_count=50;
+ int cp_op= g_the_gpu->checkpoint_option;
+ int cp_CTA = g_the_gpu->checkpoint_CTA;
+ int cp_kernel= g_the_gpu->checkpoint_kernel;
+ cp_count= g_the_gpu->checkpoint_insn_Y;
+ cp_cta_resume= g_the_gpu->checkpoint_CTA_t;
+ int cta_launched =0;
//we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise
while(!kernel.no_more_ctas_to_run()){
- functionalCoreSim cta(
- &kernel,
- g_the_gpu,
- g_the_gpu->getShaderCoreConfig()->warp_size
- );
- cta.execute();
+ unsigned temp=kernel.get_next_cta_id_single();
+
-#if (CUDART_VERSION >= 5000)
- launch_all_device_kernels();
-#endif
+ if(cp_op==0 || (cp_op==1 && cta_launched<cp_cta_resume && kernel.get_uid()==cp_kernel) || kernel.get_uid()< cp_kernel) // just fro testing
+ {
+ functionalCoreSim cta(
+ &kernel,
+ g_the_gpu,
+ g_the_gpu->getShaderCoreConfig()->warp_size
+ );
+ cta.execute(cp_count,temp);
+
+ #if (CUDART_VERSION >= 5000)
+ launch_all_device_kernels();
+ #endif
+ }
+ else
+ {
+ kernel.increment_cta_id();
+ }
+ cta_launched++;
}
+
+
+
+ if(cp_op==1)
+ {
+ char f1name[2048];
+ snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() );
+ g_checkpoint->store_global_mem(g_the_gpu->get_global_memory(), f1name , "%08x");
+ }
+
+
+
//registering this kernel as done
@@ -1874,9 +2251,10 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL )
fflush(stdout);
}
-void functionalCoreSim::initializeCTA()
+void functionalCoreSim::initializeCTA(unsigned ctaid_cp)
{
int ctaLiveThreads=0;
+ symbol_table * symtab= m_kernel->entry()->get_symtab();
for(int i=0; i< m_warp_count; i++){
m_warpAtBarrier[i]=false;
@@ -1889,9 +2267,13 @@ void functionalCoreSim::initializeCTA()
for(unsigned i=0; i<m_kernel->threads_per_cta();i++) {
ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true);
assert(m_thread[i]!=NULL && !m_thread[i]->is_done());
+ char fname[2048];
+ snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i );
+ if(cp_cta_resume==1)
+ m_thread[i]->resume_reg_thread(fname,symtab);
ctaLiveThreads++;
}
-
+
for(int k=0;k<m_warp_count;k++)
createWarp(k);
}
@@ -1908,27 +2290,91 @@ void functionalCoreSim::createWarp(unsigned warpId)
assert(m_thread[warpId*m_warp_size]!=NULL);
m_simt_stack[warpId]->launch(m_thread[warpId*m_warp_size]->get_pc(),initialMask);
+ char fname[2048];
+ snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId );
+
+ if(cp_cta_resume==1)
+ {
+ unsigned pc,rpc;
+ m_simt_stack[warpId]->resume(fname);
+ m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc);
+ for(int i=warpId*m_warp_size; i<warpId*m_warp_size+m_warp_size;i++){
+ m_thread[i]->set_npc(pc);
+ m_thread[i]->update_pc();
+ }
+
+ }
m_liveThreadCount[warpId]= liveThreadsCount;
}
-void functionalCoreSim::execute()
+void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp)
{
- initializeCTA();
+ cp_count= m_gpu->checkpoint_insn_Y;
+ cp_cta_resume= m_gpu->checkpoint_CTA_t;
+ initializeCTA(ctaid_cp);
- //start executing the CTA
+ int count=0;
while(true){
bool someOneLive= false;
bool allAtBarrier = true;
for(unsigned i=0;i<m_warp_count;i++){
executeWarp(i,allAtBarrier,someOneLive);
+ count++;
}
+
+ if(inst_count>0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option==1)
+ {
+ someOneLive=false;
+ break;
+ }
if(!someOneLive) break;
if(allAtBarrier){
for(unsigned i=0;i<m_warp_count;i++)
m_warpAtBarrier[i]=false;
}
}
- }
+
+ checkpoint *g_checkpoint;
+ g_checkpoint = new checkpoint();
+
+ symbol * sym;
+ ptx_reg_t regval;
+ regval.u64= 123;
+ symbol_table * symtab= m_kernel->entry()->get_symtab();
+
+
+ unsigned ctaid =m_kernel->get_next_cta_id_single();
+ if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t))
+ {
+ char fname[2048];
+ snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 );
+ g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , "%08x");
+ for(int i=0; i<32*m_warp_count;i++)
+ {
+ char fname[2048];
+ snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 );
+ m_thread[i]->print_reg_thread(fname);
+ char f1name[2048];
+ snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 );
+ g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , "%08x");
+ m_thread[i]->set_done();
+ m_thread[i]->exitCore();
+ m_thread[i]->registerExit();
+ }
+
+ for(int i=0;i<m_warp_count;i++)
+ {
+
+ char fname[2048];
+ snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i,ctaid-1 );
+ FILE * fp = fopen(fname,"w");
+ assert(fp!=NULL);
+ m_simt_stack[i]->print_checkpoint(fp);
+ fclose(fp);
+ }
+ }
+
+}
void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive)
{
@@ -2091,7 +2537,7 @@ struct rec_pts {
int s_num_recon;
};
-struct std::map<function_info*,rec_pts> g_rpts;
+class std::map<function_info*,rec_pts> g_rpts;
struct rec_pts find_reconvergence_points( function_info *finfo )
{
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index 958daba..e690356 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -32,6 +32,7 @@
#include"../gpgpu-sim/shader.h"
#include <stdlib.h>
#include <map>
+#include <vector>
#include <string>
#include"ptx_sim.h"
@@ -46,6 +47,8 @@ extern int g_debug_thread_uid;
extern void ** g_inst_classification_stat;
extern void ** g_inst_op_classification_stat;
extern int g_ptx_kernel_count; // used for classification stat collection purposes
+extern char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor;
+
void ptx_opcocde_latency_options (option_parser_t opp);
extern class kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
@@ -97,7 +100,7 @@ public:
delete[] m_warpAtBarrier;
}
//! executes all warps till completion
- void execute();
+ void execute(int inst_count, unsigned ctaid_cp);
virtual void warp_exit( unsigned warp_id );
virtual bool warp_waiting_at_barrier( unsigned warp_id ) const
{
@@ -107,7 +110,7 @@ public:
private:
void executeWarp(unsigned, bool &, bool &);
//initializes threads in the CTA block which we are executing
- void initializeCTA();
+ void initializeCTA(unsigned ctaid_cp);
virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid)
{
if(m_thread[tid]==NULL || m_thread[tid]->is_done()){
diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc
index 4a8ffe5..917e7a8 100644
--- a/src/cuda-sim/cuda_device_runtime.cc
+++ b/src/cuda-sim/cuda_device_runtime.cc
@@ -177,6 +177,20 @@ void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info *
//device_grid = op.grid;
device_kernel_entry = config.entry;
DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name());
+
+ //PDOM analysis is done for Parent kernel but not for child kernel.
+ if (device_kernel_entry->is_pdom_set()) {
+ printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() );
+ } else {
+ printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() );
+ /*
+ * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call.
+ * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash.
+ */
+ if (device_kernel_entry->get_function_size() >0)
+ device_kernel_entry->do_pdom();
+ device_kernel_entry->set_pdom();
+ }
//copy data in parameter_buffer to device kernel param memory
unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size();
@@ -184,7 +198,8 @@ void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info *
memory_space *device_kernel_param_mem;
//create child kernel_info_t and index it with parameter_buffer address
- device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry);
+ gpgpu_t* gpu=thread->get_gpu();
+ device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping());
device_grid->launch_cycle = gpu_sim_cycle + gpu_tot_sim_cycle;
kernel_info_t & parent_grid = thread->get_kernel();
DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" <<
diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h
new file mode 100644
index 0000000..8f1a8eb
--- /dev/null
+++ b/src/cuda-sim/half.h
@@ -0,0 +1,3067 @@
+// half - IEEE 754-based half-precision floating point library.
+//
+// Copyright (c) 2012-2017 Christian Rau <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
+// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// Version 1.12.0
+
+/// \file
+/// Main header file for half precision functionality.
+
+#ifndef HALF_HALF_HPP
+#define HALF_HALF_HPP
+
+/// Combined gcc version number.
+#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__)
+
+//check C++11 language features
+#if defined(__clang__) //clang
+ #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+/*#elif defined(__INTEL_COMPILER) //Intel C++
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ????????
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ????????
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ????????
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif*/
+#elif defined(__GNUC__) //gcc
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #endif
+#elif defined(_MSC_VER) //Visual C++
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #define HALF_POP_WARNINGS 1
+ #pragma warning(push)
+ #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned
+#endif
+
+//check C++11 library features
+#include <utility>
+#if defined(_LIBCPP_VERSION) //libc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+#elif defined(__GLIBCXX__) //libstdc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifdef __clang__
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #else
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #endif
+#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++
+ #if _CPPLIB_VER >= 520
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #if _CPPLIB_VER >= 610
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #endif
+#endif
+#undef HALF_GNUC_VERSION
+
+//support constexpr
+#if HALF_ENABLE_CPP11_CONSTEXPR
+ #define HALF_CONSTEXPR constexpr
+ #define HALF_CONSTEXPR_CONST constexpr
+#else
+ #define HALF_CONSTEXPR
+ #define HALF_CONSTEXPR_CONST const
+#endif
+
+//support noexcept
+#if HALF_ENABLE_CPP11_NOEXCEPT
+ #define HALF_NOEXCEPT noexcept
+ #define HALF_NOTHROW noexcept
+#else
+ #define HALF_NOEXCEPT
+ #define HALF_NOTHROW throw()
+#endif
+
+#include <algorithm>
+#include <iostream>
+#include <limits>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#if HALF_ENABLE_CPP11_TYPE_TRAITS
+ #include <type_traits>
+#endif
+#if HALF_ENABLE_CPP11_CSTDINT
+ #include <cstdint>
+#endif
+#if HALF_ENABLE_CPP11_HASH
+ #include <functional>
+#endif
+
+
+/// Default rounding mode.
+/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as
+/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one
+/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`:
+///
+/// `std::float_round_style` | value | rounding
+/// ---------------------------------|-------|-------------------------
+/// `std::round_indeterminate` | -1 | fastest (default)
+/// `std::round_toward_zero` | 0 | toward zero
+/// `std::round_to_nearest` | 1 | to nearest
+/// `std::round_toward_infinity` | 2 | toward positive infinity
+/// `std::round_toward_neg_infinity` | 3 | toward negative infinity
+///
+/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows
+/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits<float>::round_style`
+/// to synchronize the rounding mode with that of the underlying single-precision implementation.
+#ifndef HALF_ROUND_STYLE
+ #define HALF_ROUND_STYLE -1 // = std::round_indeterminate
+#endif
+
+/// Tie-breaking behaviour for round to nearest.
+/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is
+/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and
+/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant
+/// behaviour is needed.
+#ifndef HALF_ROUND_TIES_TO_EVEN
+ #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
+#endif
+
+/// Value signaling overflow.
+/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to a positive value signaling the overflow of an
+/// operation, in particular it just evaluates to positive infinity.
+#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
+
+/// Fast half-precision fma function.
+/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate
+/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all
+/// arithmetic operations, this is in fact always the case.
+#define FP_FAST_FMAH 1
+
+#ifndef FP_ILOGB0
+ #define FP_ILOGB0 INT_MIN
+#endif
+#ifndef FP_ILOGBNAN
+ #define FP_ILOGBNAN INT_MAX
+#endif
+#ifndef FP_SUBNORMAL
+ #define FP_SUBNORMAL 0
+#endif
+#ifndef FP_ZERO
+ #define FP_ZERO 1
+#endif
+#ifndef FP_NAN
+ #define FP_NAN 2
+#endif
+#ifndef FP_INFINITE
+ #define FP_INFINITE 3
+#endif
+#ifndef FP_NORMAL
+ #define FP_NORMAL 4
+#endif
+
+
+/// Main namespace for half precision functionality.
+/// This namespace contains all the functionality provided by the library.
+namespace half_float
+{
+ class half;
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ /// Library-defined half-precision literals.
+ /// Import this namespace to enable half-precision floating point literals:
+ /// ~~~~{.cpp}
+ /// using namespace half_float::literal;
+ /// half_float::half = 4.2_h;
+ /// ~~~~
+ namespace literal
+ {
+ half operator"" _h(long double);
+ }
+#endif
+
+ /// \internal
+ /// \brief Implementation details.
+ namespace detail
+ {
+ #if HALF_ENABLE_CPP11_TYPE_TRAITS
+ /// Conditional type.
+ template<bool B,typename T,typename F> struct conditional : std::conditional<B,T,F> {};
+
+ /// Helper for tag dispatching.
+ template<bool B> struct bool_type : std::integral_constant<bool,B> {};
+ using std::true_type;
+ using std::false_type;
+
+ /// Type traits for floating point types.
+ template<typename T> struct is_float : std::is_floating_point<T> {};
+ #else
+ /// Conditional type.
+ template<bool,typename T,typename> struct conditional { typedef T type; };
+ template<typename T,typename F> struct conditional<false,T,F> { typedef F type; };
+
+ /// Helper for tag dispatching.
+ template<bool> struct bool_type {};
+ typedef bool_type<true> true_type;
+ typedef bool_type<false> false_type;
+
+ /// Type traits for floating point types.
+ template<typename> struct is_float : false_type {};
+ template<typename T> struct is_float<const T> : is_float<T> {};
+ template<typename T> struct is_float<volatile T> : is_float<T> {};
+ template<typename T> struct is_float<const volatile T> : is_float<T> {};
+ template<> struct is_float<float> : true_type {};
+ template<> struct is_float<double> : true_type {};
+ template<> struct is_float<long double> : true_type {};
+ #endif
+
+ /// Type traits for floating point bits.
+ template<typename T> struct bits { typedef unsigned char type; };
+ template<typename T> struct bits<const T> : bits<T> {};
+ template<typename T> struct bits<volatile T> : bits<T> {};
+ template<typename T> struct bits<const volatile T> : bits<T> {};
+
+ #if HALF_ENABLE_CPP11_CSTDINT
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef std::uint_least16_t uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> { typedef std::uint_least32_t type; };
+
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef std::uint_least64_t type; };
+ #else
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef unsigned short uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> : conditional<std::numeric_limits<unsigned int>::digits>=32,unsigned int,unsigned long> {};
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> : conditional<std::numeric_limits<unsigned long>::digits>=64,unsigned long,unsigned long long> {};
+ #else
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef unsigned long type; };
+ #endif
+ #endif
+
+ /// Tag type for binary construction.
+ struct binary_t {};
+
+ /// Tag for binary construction.
+ HALF_CONSTEXPR_CONST binary_t binary = binary_t();
+
+ /// Temporary half-precision expression.
+ /// This class represents a half-precision expression which just stores a single-precision value internally.
+ struct expr
+ {
+ /// Conversion constructor.
+ /// \param f single-precision value to convert
+ explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
+
+ private:
+ /// Internal expression value stored in single-precision.
+ float value_;
+ };
+
+ /// SFINAE helper for generic half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member equivalent to \a T.
+ /// \tparam T type to return
+ template<typename T,typename,typename=void,typename=void> struct enable {};
+ template<typename T> struct enable<T,half,void,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,void,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,void> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,half> { typedef T type; };
+ template<typename T> struct enable<T,half,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,expr> { typedef T type; };
+
+ /// Return type for specialized generic 2-argument half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member denoting the appropriate return type.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct result : enable<expr,T,U> {};
+ template<> struct result<half,half> { typedef half type; };
+
+ /// \name Classification helpers
+ /// \{
+
+ /// Check for infinity.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if infinity
+ /// \retval false else
+ template<typename T> bool builtin_isinf(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isinf(arg);
+ #elif defined(_MSC_VER)
+ return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
+ #else
+ return arg == std::numeric_limits<T>::infinity() || arg == -std::numeric_limits<T>::infinity();
+ #endif
+ }
+
+ /// Check for NaN.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if not a number
+ /// \retval false else
+ template<typename T> bool builtin_isnan(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isnan(arg);
+ #elif defined(_MSC_VER)
+ return ::_isnan(static_cast<double>(arg)) != 0;
+ #else
+ return arg != arg;
+ #endif
+ }
+
+ /// Check sign.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if signbit set
+ /// \retval false else
+ template<typename T> bool builtin_signbit(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::signbit(arg);
+ #else
+ return arg < T() || (arg == T() && T(1)/arg < T());
+ #endif
+ }
+
+ /// \}
+ /// \name Conversion
+ /// \{
+
+ /// Convert IEEE single-precision to half-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value single-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(float value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ uint32 bits;// = *reinterpret_cast<uint32*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(float));
+/* uint16 hbits = (bits>>16) & 0x8000;
+ bits &= 0x7FFFFFFF;
+ int exp = bits >> 23;
+ if(exp == 255)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0x7FFFFF)!=0));
+ if(exp > 142)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s;
+ if(exp > 112)
+ {
+ g = (bits>>12) & 1;
+ s = (bits&0xFFF) != 0;
+ hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF);
+ }
+ else if(exp > 101)
+ {
+ int i = 125 - exp;
+ bits = (bits&0x7FFFFF) | 0x800000;
+ g = (bits>>i) & 1;
+ s = (bits&((1L<<i)-1)) != 0;
+ hbits |= bits >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s = bits != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+*/ static const uint16 base_table[512] = {
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
+ 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
+ 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
+ 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
+ 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 };
+ static const unsigned char shift_table[512] = {
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 };
+ uint16 hbits = base_table[bits>>23] + static_cast<uint16>((bits&0x7FFFFF)>>shift_table[bits>>23]);
+ if(R == std::round_to_nearest)
+ hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00)
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((static_cast<uint32>(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits)
+ #endif
+ ;
+ else if(R == std::round_toward_zero)
+ hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23];
+ else if(R == std::round_toward_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)&
+ ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511));
+ else if(R == std::round_toward_neg_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)&
+ ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255));
+ return hbits;
+ }
+
+ /// Convert IEEE double-precision to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value double-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(double value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint64 bits;// = *reinterpret_cast<uint64*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(double));
+ uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
+ uint16 hbits = (hi>>16) & 0x8000;
+ hi &= 0x7FFFFFFF;
+ int exp = hi >> 20;
+ if(exp == 2047)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0xFFFFFFFFFFFFF)!=0));
+ if(exp > 1038)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s = lo != 0;
+ if(exp > 1008)
+ {
+ g = (hi>>9) & 1;
+ s |= (hi&0x1FF) != 0;
+ hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF);
+ }
+ else if(exp > 997)
+ {
+ int i = 1018 - exp;
+ hi = (hi&0xFFFFF) | 0x100000;
+ g = (hi>>i) & 1;
+ s |= (hi&((1L<<i)-1)) != 0;
+ hbits |= hi >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s |= hi != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+ return hbits;
+ }
+
+ /// Convert non-IEEE floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half_impl(T value, ...)
+ {
+ uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
+ if(value == T())
+ return hbits;
+ if(builtin_isnan(value))
+ return hbits | 0x7FFF;
+ if(builtin_isinf(value))
+ return hbits | 0x7C00;
+ int exp;
+ std::frexp(value, &exp);
+ if(exp > 16)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ else if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ if(exp < -13)
+ value = std::ldexp(value, 24);
+ else
+ {
+ value = std::ldexp(value, 11-exp);
+ hbits |= ((exp+13)<<10);
+ }
+ T ival, frac = std::modf(value, &ival);
+ hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
+ if(R == std::round_to_nearest)
+ {
+ frac = std::abs(frac);
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits);
+ #else
+ hbits += frac >= T(0.5);
+ #endif
+ }
+ else if(R == std::round_toward_infinity)
+ hbits += frac > T();
+ else if(R == std::round_toward_neg_infinity)
+ hbits += frac < T();
+ return hbits;
+ }
+
+ /// Convert floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half(T value)
+ {
+ return float2half_impl<R>(value, bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam S `true` if value negative, `false` else
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value non-negative integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,bool S,typename T> uint16 int2half_impl(T value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "int to half conversion only supports builtin integer types");
+ #endif
+ if(S)
+ value = -value;
+ uint16 bits = S << 15;
+ if(value > 0xFFFF)
+ {
+ if(R == std::round_toward_infinity)
+ bits |= 0x7C00 - S;
+ else if(R == std::round_toward_neg_infinity)
+ bits |= 0x7BFF + S;
+ else
+ bits |= 0x7BFF + (R!=std::round_toward_zero);
+ }
+ else if(value)
+ {
+ unsigned int m = value, exp = 24;
+ for(; m<0x400; m<<=1,--exp) ;
+ for(; m>0x7FF; m>>=1,++exp) ;
+ bits |= (exp<<10) + m;
+ if(exp > 24)
+ {
+ if(R == std::round_to_nearest)
+ bits += (value>>(exp-25)) & 1
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((1<<(exp-25))-1)&value)!=0)|bits)
+ #endif
+ ;
+ else if(R == std::round_toward_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & !S;
+ else if(R == std::round_toward_neg_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & S;
+ }
+ }
+ return bits;
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 int2half(T value)
+ {
+ return (value<0) ? int2half_impl<R,true>(value) : int2half_impl<R,false>(value);
+ }
+
+ /// Convert half-precision to IEEE single-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \param value binary representation of half-precision value
+ /// \return single-precision value
+ inline float half2float_impl(uint16 value, float, true_type)
+ {
+ typedef bits<float>::type uint32;
+/* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ bits |= 0x38000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,bits-=0x800000) ;
+ bits += static_cast<uint32>(abs) << 13;
+ }
+*/ static const uint32 mantissa_table[2048] = {
+ 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000,
+ 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000,
+ 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
+ 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000,
+ 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
+ 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
+ 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000,
+ 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000,
+ 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
+ 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
+ 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000,
+ 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
+ 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000,
+ 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000,
+ 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
+ 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000,
+ 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000,
+ 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
+ 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000,
+ 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
+ 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
+ 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000,
+ 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000,
+ 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
+ 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
+ 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000,
+ 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
+ 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000,
+ 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000,
+ 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
+ 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000,
+ 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000,
+ 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
+ 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000,
+ 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
+ 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
+ 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000,
+ 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000,
+ 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
+ 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
+ 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000,
+ 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
+ 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000,
+ 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000,
+ 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
+ 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000,
+ 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000,
+ 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
+ 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000,
+ 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
+ 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
+ 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000,
+ 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000,
+ 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
+ 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
+ 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000,
+ 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
+ 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000,
+ 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000,
+ 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
+ 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000,
+ 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000,
+ 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
+ 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000,
+ 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
+ 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
+ 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000,
+ 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000,
+ 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
+ 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
+ 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000,
+ 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
+ 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000,
+ 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000,
+ 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
+ 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000,
+ 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000,
+ 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
+ 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000,
+ 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
+ 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
+ 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000,
+ 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000,
+ 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
+ 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
+ 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000,
+ 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
+ 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000,
+ 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000,
+ 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
+ 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000,
+ 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000,
+ 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
+ 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000,
+ 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
+ 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
+ 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000,
+ 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000,
+ 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
+ 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
+ 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000,
+ 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
+ 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000,
+ 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000,
+ 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
+ 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000,
+ 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000,
+ 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
+ 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000,
+ 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
+ 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
+ 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000,
+ 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000,
+ 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
+ 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
+ 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000,
+ 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
+ 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000,
+ 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000,
+ 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
+ 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000,
+ 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000,
+ 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
+ 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000,
+ 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
+ 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
+ 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000,
+ 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 };
+ static const uint32 exponent_table[64] = {
+ 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000,
+ 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000,
+ 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
+ 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 };
+ static const unsigned short offset_table[64] = {
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 };
+ uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];
+// return *reinterpret_cast<float*>(&bits); //violating strict aliasing!
+ float out;
+ std::memcpy(&out, &bits, sizeof(float));
+ return out;
+ }
+
+ /// Convert half-precision to IEEE double-precision.
+ /// \param value binary representation of half-precision value
+ /// \return double-precision value
+ inline double half2float_impl(uint16 value, double, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint32 hi = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ hi |= 0x3F000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,hi-=0x100000) ;
+ hi += static_cast<uint32>(abs) << 10;
+ }
+ uint64 bits = static_cast<uint64>(hi) << 32;
+// return *reinterpret_cast<double*>(&bits); //violating strict aliasing!
+ double out;
+ std::memcpy(&out, &bits, sizeof(double));
+ return out;
+ }
+
+ /// Convert half-precision to non-IEEE floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float_impl(uint16 value, T, ...)
+ {
+ T out;
+ int abs = value & 0x7FFF;
+ if(abs > 0x7C00)
+ out = std::numeric_limits<T>::has_quiet_NaN ? std::numeric_limits<T>::quiet_NaN() : T();
+ else if(abs == 0x7C00)
+ out = std::numeric_limits<T>::has_infinity ? std::numeric_limits<T>::infinity() : std::numeric_limits<T>::max();
+ else if(abs > 0x3FF)
+ out = std::ldexp(static_cast<T>((abs&0x3FF)|0x400), (abs>>10)-25);
+ else
+ out = std::ldexp(static_cast<T>(abs), -24);
+ return (value&0x8000) ? -out : out;
+ }
+
+ /// Convert half-precision to floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float(uint16 value)
+ {
+ return half2float_impl(value, T(), bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,bool E,typename T> T half2int_impl(uint16 value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "half to int conversion only supports builtin integer types");
+ #endif
+ unsigned int e = value & 0x7FFF;
+ if(e >= 0x7C00)
+ return (value&0x8000) ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
+ if(e < 0x3800)
+ {
+ if(R == std::round_toward_infinity)
+ return T(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ return -T(value>0x8000);
+ return T();
+ }
+ unsigned int m = (value&0x3FF) | 0x400;
+ e >>= 10;
+ if(e < 25)
+ {
+ if(R == std::round_to_nearest)
+ m += (1<<(24-e)) - (~(m>>(25-e))&E);
+ else if(R == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(25-e))-1U);
+ else if(R == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(25-e))-1U);
+ m >>= 25 - e;
+ }
+ else
+ m <<= e - 25;
+ return (value&0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,typename T> T half2int(uint16 value) { return half2int_impl<R,HALF_ROUND_TIES_TO_EVEN,T>(value); }
+
+ /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero.
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<typename T> T half2int_up(uint16 value) { return half2int_impl<std::round_to_nearest,0,T>(value); }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R,bool E> uint16 round_half_impl(uint16 value)
+ {
+ unsigned int e = value & 0x7FFF;
+ uint16 result = value;
+ if(e < 0x3C00)
+ {
+ result &= 0x8000;
+ if(R == std::round_to_nearest)
+ result |= 0x3C00U & -(e>=(0x3800+E));
+ else if(R == std::round_toward_infinity)
+ result |= 0x3C00U & -(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ result |= 0x3C00U & -(value>0x8000);
+ }
+ else if(e < 0x6400)
+ {
+ e = 25 - (e>>10);
+ unsigned int mask = (1<<e) - 1;
+ if(R == std::round_to_nearest)
+ result += (1<<(e-1)) - (~(result>>e)&E);
+ else if(R == std::round_toward_infinity)
+ result += mask & ((value>>15)-1);
+ else if(R == std::round_toward_neg_infinity)
+ result += mask & -(value>>15);
+ result &= ~mask;
+ }
+ return result;
+ }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R> uint16 round_half(uint16 value) { return round_half_impl<R,HALF_ROUND_TIES_TO_EVEN>(value); }
+
+ /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero.
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ inline uint16 round_half_up(uint16 value) { return round_half_impl<std::round_to_nearest,0>(value); }
+ /// \}
+
+ struct functions;
+ template<typename> struct unary_specialized;
+ template<typename,typename> struct binary_specialized;
+ template<typename,typename,std::float_round_style> struct half_caster;
+ }
+
+ /// Half-precision floating point type.
+ /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and
+ /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and
+ /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations
+ /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to
+ /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic
+ /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type).
+ ///
+ /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and
+ /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which
+ /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the
+ /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of
+ /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most
+ /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit
+ /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if
+ /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on
+ /// nearly any reasonable platform.
+ ///
+ /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable
+ /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation.
+ class half
+ {
+ friend struct detail::functions;
+ friend struct detail::unary_specialized<half>;
+ friend struct detail::binary_specialized<half,half>;
+ template<typename,typename,std::float_round_style> friend struct detail::half_caster;
+ friend class std::numeric_limits<half>;
+ #if HALF_ENABLE_CPP11_HASH
+ friend struct std::hash<half>;
+ #endif
+ #if HALF_ENABLE_CPP11_USER_LITERALS
+ friend half literal::operator"" _h(long double);
+ #endif
+
+ public:
+ /// Default constructor.
+ /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics
+ /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics.
+ HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
+
+ /// Copy constructor.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ half(detail::expr rhs) : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
+
+ /// Conversion constructor.
+ /// \param rhs float to convert
+ explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ operator float() const { return detail::half2float<float>(data_); }
+
+ /// Assignment operator.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ /// \return reference to this half
+ half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to add
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator+=(T rhs) { return *this += static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to subtract
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator-=(T rhs) { return *this -= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to multiply with
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator*=(T rhs) { return *this *= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to divide by
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator/=(T rhs) { return *this /= static_cast<float>(rhs); }
+
+ /// Assignment operator.
+ /// \param rhs single-precision value to copy from
+ /// \return reference to this half
+ half& operator=(float rhs) { data_ = detail::float2half<round_style>(rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to add
+ /// \return reference to this half
+ half& operator+=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)+rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to subtract
+ /// \return reference to this half
+ half& operator-=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)-rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to multiply with
+ /// \return reference to this half
+ half& operator*=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)*rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to divide by
+ /// \return reference to this half
+ half& operator/=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)/rhs); return *this; }
+
+ /// Prefix increment.
+ /// \return incremented half value
+ half& operator++() { return *this += 1.0f; }
+
+ /// Prefix decrement.
+ /// \return decremented half value
+ half& operator--() { return *this -= 1.0f; }
+
+ /// Postfix increment.
+ /// \return non-incremented half value
+ half operator++(int) { half out(*this); ++*this; return out; }
+
+ /// Postfix decrement.
+ /// \return non-decremented half value
+ half operator--(int) { half out(*this); --*this; return out; }
+
+ private:
+ /// Rounding mode to use
+ static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE);
+
+ /// Constructor.
+ /// \param bits binary representation to set half to
+ HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {}
+
+ /// Internal binary representation
+ detail::uint16 data_;
+ };
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ namespace literal
+ {
+ /// Half literal.
+ /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due
+ /// to rather involved conversions.
+ /// \param value literal value
+ /// \return half with given value (if representable)
+ inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half<half::round_style>(value)); }
+ }
+#endif
+
+ namespace detail
+ {
+ /// Wrapper implementing unspecialized half-precision functions.
+ struct functions
+ {
+ /// Addition implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision sum stored in single-precision
+ static expr plus(float x, float y) { return expr(x+y); }
+
+ /// Subtraction implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision difference stored in single-precision
+ static expr minus(float x, float y) { return expr(x-y); }
+
+ /// Multiplication implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision product stored in single-precision
+ static expr multiplies(float x, float y) { return expr(x*y); }
+
+ /// Division implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision quotient stored in single-precision
+ static expr divides(float x, float y) { return expr(x/y); }
+
+ /// Output implementation.
+ /// \param out stream to write to
+ /// \param arg value to write
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_ostream<charT,traits>& write(std::basic_ostream<charT,traits> &out, float arg) { return out << arg; }
+
+ /// Input implementation.
+ /// \param in stream to read from
+ /// \param arg half to read into
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_istream<charT,traits>& read(std::basic_istream<charT,traits> &in, half &arg)
+ {
+ float f;
+ if(in >> f)
+ arg = f;
+ return in;
+ }
+
+ /// Modulo implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remainder(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remainder(x, y));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return expr(builtin_signbit(x) ? -0.0f : 0.0f);
+ ax = std::fmod(ax, ay+ay);
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ if(ax >= y2)
+ ax -= ay;
+ }
+ return expr(builtin_signbit(x) ? -ax : ax);
+ #endif
+ }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store quotient bits at
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remquo(float x, float y, int *quo)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remquo(x, y, quo));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ bool sign = builtin_signbit(x), qsign = static_cast<bool>(sign^builtin_signbit(y));
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
+ ax = std::fmod(ax, 8.0f*ay);
+ int cquo = 0;
+ if(ax >= 4.0f * ay)
+ {
+ ax -= 4.0f * ay;
+ cquo += 4;
+ }
+ if(ax >= 2.0f * ay)
+ {
+ ax -= 2.0f * ay;
+ cquo += 2;
+ }
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ ++cquo;
+ if(ax >= y2)
+ {
+ ax -= ay;
+ ++cquo;
+ }
+ }
+ return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
+ #endif
+ }
+
+ /// Positive difference implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Positive difference stored in single-precision
+ static expr fdim(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fdim(x, y));
+ #else
+ return expr((x<=y) ? 0.0f : (x-y));
+ #endif
+ }
+
+ /// Fused multiply-add implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return \a x * \a y + \a z stored in single-precision
+ static expr fma(float x, float y, float z)
+ {
+ #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
+ return expr(std::fma(x, y, z));
+ #else
+ return expr(x*y+z);
+ #endif
+ }
+
+ /// Get NaN.
+ /// \return Half-precision quiet NaN
+ static half nanh() { return half(binary, 0x7FFF); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp(float arg) { return expr(std::exp(arg)); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr expm1(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::expm1(arg));
+ #else
+ return expr(static_cast<float>(std::exp(static_cast<double>(arg))-1.0));
+ #endif
+ }
+
+ /// Binary exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::exp2(arg));
+ #else
+ return expr(static_cast<float>(std::exp(arg*0.69314718055994530941723212145818)));
+ #endif
+ }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log(float arg) { return expr(std::log(arg)); }
+
+ /// Common logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log10(float arg) { return expr(std::log10(arg)); }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log1p(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log1p(arg));
+ #else
+ return expr(static_cast<float>(std::log(1.0+arg)));
+ #endif
+ }
+
+ /// Binary logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log2(arg));
+ #else
+ return expr(static_cast<float>(std::log(static_cast<double>(arg))*1.4426950408889634073599246810019));
+ #endif
+ }
+
+ /// Square root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
+
+ /// Cubic root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cbrt(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::cbrt(arg));
+ #else
+ if(builtin_isnan(arg) || builtin_isinf(arg))
+ return expr(arg);
+ return expr(builtin_signbit(arg) ? -static_cast<float>(std::pow(-static_cast<double>(arg), 1.0/3.0)) :
+ static_cast<float>(std::pow(static_cast<double>(arg), 1.0/3.0)));
+ #endif
+ }
+
+ /// Hypotenuse implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr hypot(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::hypot(x, y));
+ #else
+ return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits<float>::infinity() :
+ static_cast<float>(std::sqrt(static_cast<double>(x)*x+static_cast<double>(y)*y)));
+ #endif
+ }
+
+ /// Power implementation.
+ /// \param base value to exponentiate
+ /// \param exp power to expontiate to
+ /// \return function value stored in single-preicision
+ static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
+
+ /// Sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sin(float arg) { return expr(std::sin(arg)); }
+
+ /// Cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cos(float arg) { return expr(std::cos(arg)); }
+
+ /// Tan implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tan(float arg) { return expr(std::tan(arg)); }
+
+ /// Arc sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asin(float arg) { return expr(std::asin(arg)); }
+
+ /// Arc cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acos(float arg) { return expr(std::acos(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atan(float arg) { return expr(std::atan(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
+
+ /// Hyperbolic sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sinh(float arg) { return expr(std::sinh(arg)); }
+
+ /// Hyperbolic cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cosh(float arg) { return expr(std::cosh(arg)); }
+
+ /// Hyperbolic tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tanh(float arg) { return expr(std::tanh(arg)); }
+
+ /// Hyperbolic area sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asinh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::asinh(arg));
+ #else
+ return expr((arg==-std::numeric_limits<float>::infinity()) ? arg : static_cast<float>(std::log(arg+std::sqrt(arg*arg+1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acosh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::acosh(arg));
+ #else
+ return expr((arg<-1.0f) ? std::numeric_limits<float>::quiet_NaN() : static_cast<float>(std::log(arg+std::sqrt(arg*arg-1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atanh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::atanh(arg));
+ #else
+ return expr(static_cast<float>(0.5*std::log((1.0+arg)/(1.0-arg))));
+ #endif
+ }
+
+ /// Error function implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erf(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erf(arg));
+ #else
+ return expr(static_cast<float>(erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Complementary implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erfc(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erfc(arg));
+ #else
+ return expr(static_cast<float>(1.0-erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr lgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::lgamma(arg));
+ #else
+ if(builtin_isinf(arg))
+ return expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::infinity());
+ return expr(static_cast<float>(1.1447298858494001741434273513531-
+ std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg)));
+ }
+ return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::tgamma(arg));
+ #else
+ if(arg == 0.0f)
+ return builtin_signbit(arg) ? expr(-std::numeric_limits<float>::infinity()) : expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg)));
+ return expr(static_cast<float>((std::fmod(i, 2.0f)==0.0f) ? -value : value));
+ }
+ if(builtin_isinf(arg))
+ return expr(arg);
+ return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
+ #endif
+ }
+
+ /// Floor implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half floor(half arg) { return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_)); }
+
+ /// Ceiling implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half ceil(half arg) { return half(binary, round_half<std::round_toward_infinity>(arg.data_)); }
+
+ /// Truncation implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half trunc(half arg) { return half(binary, round_half<std::round_toward_zero>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half rint(half arg) { return half(binary, round_half<half::round_style>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lrint(half arg) { return detail::half2int<half::round_style,long>(arg.data_); }
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llround(half arg) { return detail::half2int_up<long long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llrint(half arg) { return detail::half2int<half::round_style,long long>(arg.data_); }
+ #endif
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return normalized significant
+ static half frexp(half arg, int *exp)
+ {
+ int m = arg.data_ & 0x7FFF, e = -14;
+ if(m >= 0x7C00 || !m)
+ return *exp = 0, arg;
+ for(; m<0x400; m<<=1,--e) ;
+ return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF));
+ }
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+ static half modf(half arg, half *iptr)
+ {
+ unsigned int e = arg.data_ & 0x7FFF;
+ if(e >= 0x6400)
+ return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00)));
+ if(e < 0x3C00)
+ return iptr->data_ = arg.data_ & 0x8000, arg;
+ e >>= 10;
+ unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask;
+ iptr->data_ = arg.data_ & ~mask;
+ if(!m)
+ return half(binary, arg.data_&0x8000);
+ for(; m<0x400; m<<=1,--e) ;
+ return half(binary, static_cast<uint16>((arg.data_&0x8000)|(e<<10)|(m&0x3FF)));
+ }
+
+ /// Scaling implementation.
+ /// \param arg number to scale
+ /// \param exp power of two to scale by
+ /// \return scaled number
+ static half scalbln(half arg, long exp)
+ {
+ unsigned int m = arg.data_ & 0x7FFF;
+ if(m >= 0x7C00 || !m)
+ return arg;
+ for(; m<0x400; m<<=1,--exp) ;
+ exp += m >> 10;
+ uint16 value = arg.data_ & 0x8000;
+ if(exp > 30)
+ {
+ if(half::round_style == std::round_toward_zero)
+ value |= 0x7BFF;
+ else if(half::round_style == std::round_toward_infinity)
+ value |= 0x7C00 - (value>>15);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value |= 0x7BFF + (value>>15);
+ else
+ value |= 0x7C00;
+ }
+ else if(exp > 0)
+ value |= (exp<<10) | (m&0x3FF);
+ else if(exp > -11)
+ {
+ m = (m&0x3FF) | 0x400;
+ if(half::round_style == std::round_to_nearest)
+ {
+ m += 1 << -exp;
+ #if HALF_ROUND_TIES_TO_EVEN
+ m -= (m>>(1-exp)) & 1;
+ #endif
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(1-exp))-1U);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(1-exp))-1U);
+ value |= m >> (1-exp);
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ value -= (value>>15) - 1;
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value += value >> 15;
+ return half(binary, value);
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static int ilogb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return FP_ILOGB0;
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ return exp;
+ }
+ if(abs > 0x7C00)
+ return FP_ILOGBNAN;
+ return INT_MAX;
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static half logb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return half(binary, 0xFC00);
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ uint16 bits = (exp<0) << 15;
+ if(exp)
+ {
+ unsigned int m = std::abs(exp) << 6, e = 18;
+ for(; m<0x400; m<<=1,--e) ;
+ bits |= (e<<10) + m;
+ }
+ return half(binary, bits);
+ }
+ if(abs > 0x7C00)
+ return arg;
+ return half(binary, 0x7C00);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nextafter(half from, half to)
+ {
+ uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
+ if(fabs > 0x7C00)
+ return from;
+ if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs))
+ return to;
+ if(!fabs)
+ return half(binary, (to.data_&0x8000)+1);
+ bool lt = ((fabs==from.data_) ? static_cast<int>(fabs) : -static_cast<int>(fabs)) <
+ ((tabs==to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lt))<<1)-1);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nexttoward(half from, long double to)
+ {
+ if(isnan(from))
+ return from;
+ long double lfrom = static_cast<long double>(from);
+ if(builtin_isnan(to) || lfrom == to)
+ return half(static_cast<float>(to));
+ if(!(from.data_&0x7FFF))
+ return half(binary, (static_cast<detail::uint16>(builtin_signbit(to))<<15)+1);
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lfrom<to))<<1)-1);
+ }
+
+ /// Sign implementation
+ /// \param x first operand
+ /// \param y second operand
+ /// \return composed value
+ static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static int fpclassify(half arg)
+ {
+ unsigned int abs = arg.data_ & 0x7FFF;
+ return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO;
+ }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if finite number
+ /// \retval false else
+ static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if not a number
+ /// \retval false else
+ static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if normal number
+ /// \retval false else
+ static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); }
+
+ /// Sign bit implementation.
+ /// \param arg value to check
+ /// \retval true if signed
+ /// \retval false if unsigned
+ static bool signbit(half arg) { return (arg.data_&0x8000) != 0; }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x > \a y
+ /// \retval false else
+ static bool isgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x >= \a y
+ /// \retval false else
+ static bool isgreaterequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x < \a y
+ /// \retval false else
+ static bool isless(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x <= \a y
+ /// \retval false else
+ static bool islessequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either \a x > \a y nor \a x < \a y
+ /// \retval false else
+ static bool islessgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00 || yabs > 0x7C00)
+ return false;
+ int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs;
+ return a < b || a > b;
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operand unordered
+ /// \retval false else
+ static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
+
+ private:
+ static double erf(double arg)
+ {
+ if(builtin_isinf(arg))
+ return (arg<0.0) ? -1.0 : 1.0;
+ double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2)));
+ return builtin_signbit(arg) ? -value : value;
+ }
+
+ static double lgamma(double arg)
+ {
+ double v = 1.0;
+ for(; arg<8.0; ++arg) v *= arg;
+ double w = 1.0 / (arg*arg);
+ return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+
+ -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+
+ -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+
+ -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg +
+ 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg);
+ }
+ };
+
+ /// Wrapper for unary half-precision functions needing specialization for individual argument types.
+ /// \tparam T argument type
+ template<typename T> struct unary_specialized
+ {
+ /// Negation implementation.
+ /// \param arg value to negate
+ /// \return negated value
+ static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); }
+
+ /// Absolute value implementation.
+ /// \param arg function argument
+ /// \return absolute value
+ static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); }
+ };
+ template<> struct unary_specialized<expr>
+ {
+ static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
+ static expr fabs(float arg) { return expr(std::fabs(arg)); }
+ };
+
+ /// Wrapper for binary half-precision functions needing specialization for individual argument types.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct binary_specialized
+ {
+ /// Minimum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum value
+ static expr fmin(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmin(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::min(x, y));
+ #endif
+ }
+
+ /// Maximum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum value
+ static expr fmax(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmax(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::max(x, y));
+ #endif
+ }
+ };
+ template<> struct binary_specialized<half,half>
+ {
+ static half fmin(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ static half fmax(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ };
+
+ /// Helper class for half casts.
+ /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member
+ /// function and a corresponding `type` member denoting its return type.
+ /// \tparam T destination type
+ /// \tparam U source type
+ /// \tparam R rounding mode to use
+ template<typename T,typename U,std::float_round_style R=(std::float_round_style)(HALF_ROUND_STYLE)> struct half_caster {};
+ template<typename U,std::float_round_style R> struct half_caster<half,U,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<U>::value, "half_cast from non-arithmetic type unsupported");
+ #endif
+
+ static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
+
+ private:
+ static half cast_impl(U arg, true_type) { return half(binary, float2half<R>(arg)); }
+ static half cast_impl(U arg, false_type) { return half(binary, int2half<R>(arg)); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,half,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,expr,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<std::float_round_style R> struct half_caster<half,half,R>
+ {
+ static half cast(half arg) { return arg; }
+ };
+ template<std::float_round_style R> struct half_caster<half,expr,R> : half_caster<half,half,R> {};
+
+ /// \name Comparison operators
+ /// \{
+
+ /// Comparison for equality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator==(T x, U y) { return functions::isequal(x, y); }
+
+ /// Comparison for inequality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator!=(T x, U y) { return functions::isnotequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<(T x, U y) { return functions::isless(x, y); }
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>(T x, U y) { return functions::isgreater(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<=(T x, U y) { return functions::islessequal(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); }
+
+ /// \}
+ /// \name Arithmetic operators
+ /// \{
+
+ /// Add halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return sum of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator+(T x, U y) { return functions::plus(x, y); }
+
+ /// Subtract halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return difference of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator-(T x, U y) { return functions::minus(x, y); }
+
+ /// Multiply halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return product of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator*(T x, U y) { return functions::multiplies(x, y); }
+
+ /// Divide halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return quotient of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator/(T x, U y) { return functions::divides(x, y); }
+
+ /// Identity.
+ /// \param arg operand
+ /// \return uncahnged operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator+(T arg) { return arg; }
+
+ /// Negation.
+ /// \param arg operand
+ /// \return negated operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator-(T arg) { return unary_specialized<T>::negate(arg); }
+
+ /// \}
+ /// \name Input and output
+ /// \{
+
+ /// Output operator.
+ /// \param out output stream to write into
+ /// \param arg half expression to write
+ /// \return reference to output stream
+ template<typename T,typename charT,typename traits> typename enable<std::basic_ostream<charT,traits>&,T>::type
+ operator<<(std::basic_ostream<charT,traits> &out, T arg) { return functions::write(out, arg); }
+
+ /// Input operator.
+ /// \param in input stream to read from
+ /// \param arg half to read into
+ /// \return reference to input stream
+ template<typename charT,typename traits> std::basic_istream<charT,traits>&
+ operator>>(std::basic_istream<charT,traits> &in, half &arg) { return functions::read(in, arg); }
+
+ /// \}
+ /// \name Basic mathematical operations
+ /// \{
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type abs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type fabs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type fmod(T x, U y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remainder(T x, U y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store some bits of quotient at
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+
+ /// Fused multiply add.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return ( \a x * \a y ) + \a z rounded as one operation.
+// template<typename T,typename U,typename V> typename enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
+
+ /// Maximum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum of operands
+// template<typename T,typename U> typename result<T,U>::type fmax(T x, U y) { return binary_specialized<T,U>::fmax(x, y); }
+ inline half fmax(half x, half y) { return binary_specialized<half,half>::fmax(x, y); }
+ inline expr fmax(half x, expr y) { return binary_specialized<half,expr>::fmax(x, y); }
+ inline expr fmax(expr x, half y) { return binary_specialized<expr,half>::fmax(x, y); }
+ inline expr fmax(expr x, expr y) { return binary_specialized<expr,expr>::fmax(x, y); }
+
+ /// Minimum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum of operands
+// template<typename T,typename U> typename result<T,U>::type fmin(T x, U y) { return binary_specialized<T,U>::fmin(x, y); }
+ inline half fmin(half x, half y) { return binary_specialized<half,half>::fmin(x, y); }
+ inline expr fmin(half x, expr y) { return binary_specialized<half,expr>::fmin(x, y); }
+ inline expr fmin(expr x, half y) { return binary_specialized<expr,half>::fmin(x, y); }
+ inline expr fmin(expr x, expr y) { return binary_specialized<expr,expr>::fmin(x, y); }
+
+ /// Positive difference.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return \a x - \a y or 0 if difference negative
+// template<typename T,typename U> typename enable<expr,T,U>::type fdim(T x, U y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
+
+ /// Get NaN value.
+ /// \return quiet NaN
+ inline half nanh(const char*) { return functions::nanh(); }
+
+ /// \}
+ /// \name Exponential functions
+ /// \{
+
+ /// Exponential function.
+ /// \param arg function argument
+ /// \return e raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp(T arg) { return functions::exp(arg); }
+ inline expr exp(half arg) { return functions::exp(arg); }
+ inline expr exp(expr arg) { return functions::exp(arg); }
+
+ /// Exponential minus one.
+ /// \param arg function argument
+ /// \return e raised to \a arg subtracted by 1
+// template<typename T> typename enable<expr,T>::type expm1(T arg) { return functions::expm1(arg); }
+ inline expr expm1(half arg) { return functions::expm1(arg); }
+ inline expr expm1(expr arg) { return functions::expm1(arg); }
+
+ /// Binary exponential.
+ /// \param arg function argument
+ /// \return 2 raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp2(T arg) { return functions::exp2(arg); }
+ inline expr exp2(half arg) { return functions::exp2(arg); }
+ inline expr exp2(expr arg) { return functions::exp2(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base e
+// template<typename T> typename enable<expr,T>::type log(T arg) { return functions::log(arg); }
+ inline expr log(half arg) { return functions::log(arg); }
+ inline expr log(expr arg) { return functions::log(arg); }
+
+ /// Common logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 10
+// template<typename T> typename enable<expr,T>::type log10(T arg) { return functions::log10(arg); }
+ inline expr log10(half arg) { return functions::log10(arg); }
+ inline expr log10(expr arg) { return functions::log10(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg plus 1 to base e
+// template<typename T> typename enable<expr,T>::type log1p(T arg) { return functions::log1p(arg); }
+ inline expr log1p(half arg) { return functions::log1p(arg); }
+ inline expr log1p(expr arg) { return functions::log1p(arg); }
+
+ /// Binary logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 2
+// template<typename T> typename enable<expr,T>::type log2(T arg) { return functions::log2(arg); }
+ inline expr log2(half arg) { return functions::log2(arg); }
+ inline expr log2(expr arg) { return functions::log2(arg); }
+
+ /// \}
+ /// \name Power functions
+ /// \{
+
+ /// Square root.
+ /// \param arg function argument
+ /// \return square root of \a arg
+// template<typename T> typename enable<expr,T>::type sqrt(T arg) { return functions::sqrt(arg); }
+ inline expr sqrt(half arg) { return functions::sqrt(arg); }
+ inline expr sqrt(expr arg) { return functions::sqrt(arg); }
+
+ /// Cubic root.
+ /// \param arg function argument
+ /// \return cubic root of \a arg
+// template<typename T> typename enable<expr,T>::type cbrt(T arg) { return functions::cbrt(arg); }
+ inline expr cbrt(half arg) { return functions::cbrt(arg); }
+ inline expr cbrt(expr arg) { return functions::cbrt(arg); }
+
+ /// Hypotenuse function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return square root of sum of squares without internal over- or underflows
+// template<typename T,typename U> typename enable<expr,T,U>::type hypot(T x, U y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
+
+ /// Power function.
+ /// \param base first argument
+ /// \param exp second argument
+ /// \return \a base raised to \a exp
+// template<typename T,typename U> typename enable<expr,T,U>::type pow(T base, U exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
+
+ /// \}
+ /// \name Trigonometric functions
+ /// \{
+
+ /// Sine function.
+ /// \param arg function argument
+ /// \return sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sin(T arg) { return functions::sin(arg); }
+ inline expr sin(half arg) { return functions::sin(arg); }
+ inline expr sin(expr arg) { return functions::sin(arg); }
+
+ /// Cosine function.
+ /// \param arg function argument
+ /// \return cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cos(T arg) { return functions::cos(arg); }
+ inline expr cos(half arg) { return functions::cos(arg); }
+ inline expr cos(expr arg) { return functions::cos(arg); }
+
+ /// Tangent function.
+ /// \param arg function argument
+ /// \return tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tan(T arg) { return functions::tan(arg); }
+ inline expr tan(half arg) { return functions::tan(arg); }
+ inline expr tan(expr arg) { return functions::tan(arg); }
+
+ /// Arc sine.
+ /// \param arg function argument
+ /// \return arc sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asin(T arg) { return functions::asin(arg); }
+ inline expr asin(half arg) { return functions::asin(arg); }
+ inline expr asin(expr arg) { return functions::asin(arg); }
+
+ /// Arc cosine function.
+ /// \param arg function argument
+ /// \return arc cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acos(T arg) { return functions::acos(arg); }
+ inline expr acos(half arg) { return functions::acos(arg); }
+ inline expr acos(expr arg) { return functions::acos(arg); }
+
+ /// Arc tangent function.
+ /// \param arg function argument
+ /// \return arc tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atan(T arg) { return functions::atan(arg); }
+ inline expr atan(half arg) { return functions::atan(arg); }
+ inline expr atan(expr arg) { return functions::atan(arg); }
+
+ /// Arc tangent function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return arc tangent value
+// template<typename T,typename U> typename enable<expr,T,U>::type atan2(T x, U y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
+
+ /// \}
+ /// \name Hyperbolic functions
+ /// \{
+
+ /// Hyperbolic sine.
+ /// \param arg function argument
+ /// \return hyperbolic sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sinh(T arg) { return functions::sinh(arg); }
+ inline expr sinh(half arg) { return functions::sinh(arg); }
+ inline expr sinh(expr arg) { return functions::sinh(arg); }
+
+ /// Hyperbolic cosine.
+ /// \param arg function argument
+ /// \return hyperbolic cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cosh(T arg) { return functions::cosh(arg); }
+ inline expr cosh(half arg) { return functions::cosh(arg); }
+ inline expr cosh(expr arg) { return functions::cosh(arg); }
+
+ /// Hyperbolic tangent.
+ /// \param arg function argument
+ /// \return hyperbolic tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tanh(T arg) { return functions::tanh(arg); }
+ inline expr tanh(half arg) { return functions::tanh(arg); }
+ inline expr tanh(expr arg) { return functions::tanh(arg); }
+
+ /// Hyperbolic area sine.
+ /// \param arg function argument
+ /// \return area sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asinh(T arg) { return functions::asinh(arg); }
+ inline expr asinh(half arg) { return functions::asinh(arg); }
+ inline expr asinh(expr arg) { return functions::asinh(arg); }
+
+ /// Hyperbolic area cosine.
+ /// \param arg function argument
+ /// \return area cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acosh(T arg) { return functions::acosh(arg); }
+ inline expr acosh(half arg) { return functions::acosh(arg); }
+ inline expr acosh(expr arg) { return functions::acosh(arg); }
+
+ /// Hyperbolic area tangent.
+ /// \param arg function argument
+ /// \return area tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atanh(T arg) { return functions::atanh(arg); }
+ inline expr atanh(half arg) { return functions::atanh(arg); }
+ inline expr atanh(expr arg) { return functions::atanh(arg); }
+
+ /// \}
+ /// \name Error and gamma functions
+ /// \{
+
+ /// Error function.
+ /// \param arg function argument
+ /// \return error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erf(T arg) { return functions::erf(arg); }
+ inline expr erf(half arg) { return functions::erf(arg); }
+ inline expr erf(expr arg) { return functions::erf(arg); }
+
+ /// Complementary error function.
+ /// \param arg function argument
+ /// \return 1 minus error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erfc(T arg) { return functions::erfc(arg); }
+ inline expr erfc(half arg) { return functions::erfc(arg); }
+ inline expr erfc(expr arg) { return functions::erfc(arg); }
+
+ /// Natural logarithm of gamma function.
+ /// \param arg function argument
+ /// \return natural logarith of gamma function for \a arg
+// template<typename T> typename enable<expr,T>::type lgamma(T arg) { return functions::lgamma(arg); }
+ inline expr lgamma(half arg) { return functions::lgamma(arg); }
+ inline expr lgamma(expr arg) { return functions::lgamma(arg); }
+
+ /// Gamma function.
+ /// \param arg function argument
+ /// \return gamma function value of \a arg
+// template<typename T> typename enable<expr,T>::type tgamma(T arg) { return functions::tgamma(arg); }
+ inline expr tgamma(half arg) { return functions::tgamma(arg); }
+ inline expr tgamma(expr arg) { return functions::tgamma(arg); }
+
+ /// \}
+ /// \name Rounding
+ /// \{
+
+ /// Nearest integer not less than half value.
+ /// \param arg half to round
+ /// \return nearest integer not less than \a arg
+// template<typename T> typename enable<half,T>::type ceil(T arg) { return functions::ceil(arg); }
+ inline half ceil(half arg) { return functions::ceil(arg); }
+ inline half ceil(expr arg) { return functions::ceil(arg); }
+
+ /// Nearest integer not greater than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater than \a arg
+// template<typename T> typename enable<half,T>::type floor(T arg) { return functions::floor(arg); }
+ inline half floor(half arg) { return functions::floor(arg); }
+ inline half floor(expr arg) { return functions::floor(arg); }
+
+ /// Nearest integer not greater in magnitude than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater in magnitude than \a arg
+// template<typename T> typename enable<half,T>::type trunc(T arg) { return functions::trunc(arg); }
+ inline half trunc(half arg) { return functions::trunc(arg); }
+ inline half trunc(expr arg) { return functions::trunc(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<half,T>::type round(T arg) { return functions::round(arg); }
+ inline half round(half arg) { return functions::round(arg); }
+ inline half round(expr arg) { return functions::round(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long,T>::type lround(T arg) { return functions::lround(arg); }
+ inline long lround(half arg) { return functions::lround(arg); }
+ inline long lround(expr arg) { return functions::lround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type nearbyint(T arg) { return functions::nearbyint(arg); }
+ inline half nearbyint(half arg) { return functions::rint(arg); }
+ inline half nearbyint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type rint(T arg) { return functions::rint(arg); }
+ inline half rint(half arg) { return functions::rint(arg); }
+ inline half rint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long,T>::type lrint(T arg) { return functions::lrint(arg); }
+ inline long lrint(half arg) { return functions::lrint(arg); }
+ inline long lrint(expr arg) { return functions::lrint(arg); }
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long long,T>::type llround(T arg) { return functions::llround(arg); }
+ inline long long llround(half arg) { return functions::llround(arg); }
+ inline long long llround(expr arg) { return functions::llround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long long,T>::type llrint(T arg) { return functions::llrint(arg); }
+ inline long long llrint(half arg) { return functions::llrint(arg); }
+ inline long long llrint(expr arg) { return functions::llrint(arg); }
+ #endif
+
+ /// \}
+ /// \name Floating point manipulation
+ /// \{
+
+ /// Decompress floating point number.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return significant in range [0.5, 1)
+// template<typename T> typename enable<half,T>::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract integer and fractional parts.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+// template<typename T> typename enable<half,T>::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+ /// \retval FP_ILOGB0 for zero
+ /// \retval FP_ILOGBNAN for NaN
+ /// \retval MAX_INT for infinity
+// template<typename T> typename enable<int,T>::type ilogb(T arg) { return functions::ilogb(arg); }
+ inline int ilogb(half arg) { return functions::ilogb(arg); }
+ inline int ilogb(expr arg) { return functions::ilogb(arg); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+// template<typename T> typename enable<half,T>::type logb(T arg) { return functions::logb(arg); }
+ inline half logb(half arg) { return functions::logb(arg); }
+ inline half logb(expr arg) { return functions::logb(arg); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T,typename U> typename enable<half,T,U>::type nextafter(T from, U to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, expr to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T> typename enable<half,T>::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); }
+
+ /// Take sign.
+ /// \param x value to change sign for
+ /// \param y value to take sign from
+ /// \return value equal to \a x in magnitude and to \a y in sign
+// template<typename T,typename U> typename enable<half,T,U>::type copysign(T x, U y) { return functions::copysign(x, y); }
+ inline half copysign(half x, half y) { return functions::copysign(x, y); }
+ inline half copysign(half x, expr y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, half y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+
+ /// \}
+ /// \name Floating point classification
+ /// \{
+
+
+ /// Classify floating point value.
+ /// \param arg number to classify
+ /// \retval FP_ZERO for positive and negative zero
+ /// \retval FP_SUBNORMAL for subnormal numbers
+ /// \retval FP_INFINITY for positive and negative infinity
+ /// \retval FP_NAN for NaNs
+ /// \retval FP_NORMAL for all other (normal) values
+// template<typename T> typename enable<int,T>::type fpclassify(T arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(half arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
+
+ /// Check if finite number.
+ /// \param arg number to check
+ /// \retval true if neither infinity nor NaN
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isfinite(T arg) { return functions::isfinite(arg); }
+ inline bool isfinite(half arg) { return functions::isfinite(arg); }
+ inline bool isfinite(expr arg) { return functions::isfinite(arg); }
+
+ /// Check for infinity.
+ /// \param arg number to check
+ /// \retval true for positive or negative infinity
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isinf(T arg) { return functions::isinf(arg); }
+ inline bool isinf(half arg) { return functions::isinf(arg); }
+ inline bool isinf(expr arg) { return functions::isinf(arg); }
+
+ /// Check for NaN.
+ /// \param arg number to check
+ /// \retval true for NaNs
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isnan(T arg) { return functions::isnan(arg); }
+ inline bool isnan(half arg) { return functions::isnan(arg); }
+ inline bool isnan(expr arg) { return functions::isnan(arg); }
+
+ /// Check if normal number.
+ /// \param arg number to check
+ /// \retval true if normal number
+ /// \retval false if either subnormal, zero, infinity or NaN
+// template<typename T> typename enable<bool,T>::type isnormal(T arg) { return functions::isnormal(arg); }
+ inline bool isnormal(half arg) { return functions::isnormal(arg); }
+ inline bool isnormal(expr arg) { return functions::isnormal(arg); }
+
+ /// Check sign.
+ /// \param arg number to check
+ /// \retval true for negative number
+ /// \retval false for positive number
+// template<typename T> typename enable<bool,T>::type signbit(T arg) { return functions::signbit(arg); }
+ inline bool signbit(half arg) { return functions::signbit(arg); }
+ inline bool signbit(expr arg) { return functions::signbit(arg); }
+
+ /// \}
+ /// \name Comparison
+ /// \{
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreater(T x, U y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isless(T x, U y) { return functions::isless(x, y); }
+ inline bool isless(half x, half y) { return functions::isless(x, y); }
+ inline bool isless(half x, expr y) { return functions::isless(x, y); }
+ inline bool isless(expr x, half y) { return functions::isless(x, y); }
+ inline bool isless(expr x, expr y) { return functions::isless(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessequal(T x, U y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
+
+ /// Comarison for less or greater.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either less or greater
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessgreater(T x, U y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); }
+
+ /// Check if unordered.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if unordered (one or two NaN operands)
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isunordered(T x, U y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
+
+ /// \name Casting
+ /// \{
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ /// It uses the default rounding mode.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,typename U> T half_cast(U arg) { return half_caster<T,U>::cast(arg); }
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam R rounding mode to use.
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,std::float_round_style R,typename U> T half_cast(U arg) { return half_caster<T,U,R>::cast(arg); }
+ /// \}
+ }
+
+ using detail::operator==;
+ using detail::operator!=;
+ using detail::operator<;
+ using detail::operator>;
+ using detail::operator<=;
+ using detail::operator>=;
+ using detail::operator+;
+ using detail::operator-;
+ using detail::operator*;
+ using detail::operator/;
+ using detail::operator<<;
+ using detail::operator>>;
+
+ using detail::abs;
+ using detail::fabs;
+ using detail::fmod;
+ using detail::remainder;
+ using detail::remquo;
+ using detail::fma;
+ using detail::fmax;
+ using detail::fmin;
+ using detail::fdim;
+ using detail::nanh;
+ using detail::exp;
+ using detail::expm1;
+ using detail::exp2;
+ using detail::log;
+ using detail::log10;
+ using detail::log1p;
+ using detail::log2;
+ using detail::sqrt;
+ using detail::cbrt;
+ using detail::hypot;
+ using detail::pow;
+ using detail::sin;
+ using detail::cos;
+ using detail::tan;
+ using detail::asin;
+ using detail::acos;
+ using detail::atan;
+ using detail::atan2;
+ using detail::sinh;
+ using detail::cosh;
+ using detail::tanh;
+ using detail::asinh;
+ using detail::acosh;
+ using detail::atanh;
+ using detail::erf;
+ using detail::erfc;
+ using detail::lgamma;
+ using detail::tgamma;
+ using detail::ceil;
+ using detail::floor;
+ using detail::trunc;
+ using detail::round;
+ using detail::lround;
+ using detail::nearbyint;
+ using detail::rint;
+ using detail::lrint;
+#if HALF_ENABLE_CPP11_LONG_LONG
+ using detail::llround;
+ using detail::llrint;
+#endif
+ using detail::frexp;
+ using detail::ldexp;
+ using detail::modf;
+ using detail::scalbn;
+ using detail::scalbln;
+ using detail::ilogb;
+ using detail::logb;
+ using detail::nextafter;
+ using detail::nexttoward;
+ using detail::copysign;
+ using detail::fpclassify;
+ using detail::isfinite;
+ using detail::isinf;
+ using detail::isnan;
+ using detail::isnormal;
+ using detail::signbit;
+ using detail::isgreater;
+ using detail::isgreaterequal;
+ using detail::isless;
+ using detail::islessequal;
+ using detail::islessgreater;
+ using detail::isunordered;
+
+ using detail::half_cast;
+}
+
+
+/// Extensions to the C++ standard library.
+namespace std
+{
+ /// Numeric limits for half-precision floats.
+ /// Because of the underlying single-precision implementation of many operations, it inherits some properties from
+ /// `std::numeric_limits<float>`.
+ template<> class numeric_limits<half_float::half> : public numeric_limits<float>
+ {
+ public:
+ /// Supports signed values.
+ static HALF_CONSTEXPR_CONST bool is_signed = true;
+
+ /// Is not exact.
+ static HALF_CONSTEXPR_CONST bool is_exact = false;
+
+ /// Doesn't provide modulo arithmetic.
+ static HALF_CONSTEXPR_CONST bool is_modulo = false;
+
+ /// IEEE conformant.
+ static HALF_CONSTEXPR_CONST bool is_iec559 = true;
+
+ /// Supports infinity.
+ static HALF_CONSTEXPR_CONST bool has_infinity = true;
+
+ /// Supports quiet NaNs.
+ static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
+
+ /// Supports subnormal values.
+ static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
+
+ /// Rounding mode.
+ /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying
+ /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding
+ /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the
+ /// single-precision rounding mode.
+ static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits<float>::round_style==
+ half_float::half::round_style) ? half_float::half::round_style : round_indeterminate;
+
+ /// Significant digits.
+ static HALF_CONSTEXPR_CONST int digits = 11;
+
+ /// Significant decimal digits.
+ static HALF_CONSTEXPR_CONST int digits10 = 3;
+
+ /// Required decimal digits to represent all possible values.
+ static HALF_CONSTEXPR_CONST int max_digits10 = 5;
+
+ /// Number base.
+ static HALF_CONSTEXPR_CONST int radix = 2;
+
+ /// One more than smallest exponent.
+ static HALF_CONSTEXPR_CONST int min_exponent = -13;
+
+ /// Smallest normalized representable power of 10.
+ static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
+
+ /// One more than largest exponent
+ static HALF_CONSTEXPR_CONST int max_exponent = 16;
+
+ /// Largest finitely representable power of 10.
+ static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
+
+ /// Smallest positive normal value.
+ static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); }
+
+ /// Smallest finite value.
+ static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); }
+
+ /// Largest finite value.
+ static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); }
+
+ /// Difference between one and next representable value.
+ static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); }
+
+ /// Maximum rounding error.
+ static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW
+ { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); }
+
+ /// Positive infinity.
+ static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); }
+
+ /// Quiet NaN.
+ static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); }
+
+ /// Signalling NaN.
+ static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); }
+
+ /// Smallest positive subnormal value.
+ static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); }
+ };
+
+#if HALF_ENABLE_CPP11_HASH
+ /// Hash function for half-precision floats.
+ /// This is only defined if C++11 `std::hash` is supported and enabled.
+ template<> struct hash<half_float::half> //: unary_function<half_float::half,size_t>
+ {
+ /// Type of function argument.
+ typedef half_float::half argument_type;
+
+ /// Function return type.
+ typedef size_t result_type;
+
+ /// Compute hash function.
+ /// \param arg half to hash
+ /// \return hash value
+ result_type operator()(argument_type arg) const
+ { return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_)&-(arg.data_!=0x8000)); }
+ };
+#endif
+}
+
+
+#undef HALF_CONSTEXPR
+#undef HALF_CONSTEXPR_CONST
+#undef HALF_NOEXCEPT
+#undef HALF_NOTHROW
+#ifdef HALF_POP_WARNINGS
+ #pragma warning(pop)
+ #undef HALF_POP_WARNINGS
+#endif
+
+#endif
diff --git a/src/cuda-sim/half.hpp b/src/cuda-sim/half.hpp
new file mode 100644
index 0000000..8f1a8eb
--- /dev/null
+++ b/src/cuda-sim/half.hpp
@@ -0,0 +1,3067 @@
+// half - IEEE 754-based half-precision floating point library.
+//
+// Copyright (c) 2012-2017 Christian Rau <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
+// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// Version 1.12.0
+
+/// \file
+/// Main header file for half precision functionality.
+
+#ifndef HALF_HALF_HPP
+#define HALF_HALF_HPP
+
+/// Combined gcc version number.
+#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__)
+
+//check C++11 language features
+#if defined(__clang__) //clang
+ #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+/*#elif defined(__INTEL_COMPILER) //Intel C++
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ????????
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ????????
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ????????
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif*/
+#elif defined(__GNUC__) //gcc
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #endif
+#elif defined(_MSC_VER) //Visual C++
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #define HALF_POP_WARNINGS 1
+ #pragma warning(push)
+ #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned
+#endif
+
+//check C++11 library features
+#include <utility>
+#if defined(_LIBCPP_VERSION) //libc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+#elif defined(__GLIBCXX__) //libstdc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifdef __clang__
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #else
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #endif
+#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++
+ #if _CPPLIB_VER >= 520
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #if _CPPLIB_VER >= 610
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #endif
+#endif
+#undef HALF_GNUC_VERSION
+
+//support constexpr
+#if HALF_ENABLE_CPP11_CONSTEXPR
+ #define HALF_CONSTEXPR constexpr
+ #define HALF_CONSTEXPR_CONST constexpr
+#else
+ #define HALF_CONSTEXPR
+ #define HALF_CONSTEXPR_CONST const
+#endif
+
+//support noexcept
+#if HALF_ENABLE_CPP11_NOEXCEPT
+ #define HALF_NOEXCEPT noexcept
+ #define HALF_NOTHROW noexcept
+#else
+ #define HALF_NOEXCEPT
+ #define HALF_NOTHROW throw()
+#endif
+
+#include <algorithm>
+#include <iostream>
+#include <limits>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#if HALF_ENABLE_CPP11_TYPE_TRAITS
+ #include <type_traits>
+#endif
+#if HALF_ENABLE_CPP11_CSTDINT
+ #include <cstdint>
+#endif
+#if HALF_ENABLE_CPP11_HASH
+ #include <functional>
+#endif
+
+
+/// Default rounding mode.
+/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as
+/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one
+/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`:
+///
+/// `std::float_round_style` | value | rounding
+/// ---------------------------------|-------|-------------------------
+/// `std::round_indeterminate` | -1 | fastest (default)
+/// `std::round_toward_zero` | 0 | toward zero
+/// `std::round_to_nearest` | 1 | to nearest
+/// `std::round_toward_infinity` | 2 | toward positive infinity
+/// `std::round_toward_neg_infinity` | 3 | toward negative infinity
+///
+/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows
+/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits<float>::round_style`
+/// to synchronize the rounding mode with that of the underlying single-precision implementation.
+#ifndef HALF_ROUND_STYLE
+ #define HALF_ROUND_STYLE -1 // = std::round_indeterminate
+#endif
+
+/// Tie-breaking behaviour for round to nearest.
+/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is
+/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and
+/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant
+/// behaviour is needed.
+#ifndef HALF_ROUND_TIES_TO_EVEN
+ #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
+#endif
+
+/// Value signaling overflow.
+/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to a positive value signaling the overflow of an
+/// operation, in particular it just evaluates to positive infinity.
+#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
+
+/// Fast half-precision fma function.
+/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate
+/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all
+/// arithmetic operations, this is in fact always the case.
+#define FP_FAST_FMAH 1
+
+#ifndef FP_ILOGB0
+ #define FP_ILOGB0 INT_MIN
+#endif
+#ifndef FP_ILOGBNAN
+ #define FP_ILOGBNAN INT_MAX
+#endif
+#ifndef FP_SUBNORMAL
+ #define FP_SUBNORMAL 0
+#endif
+#ifndef FP_ZERO
+ #define FP_ZERO 1
+#endif
+#ifndef FP_NAN
+ #define FP_NAN 2
+#endif
+#ifndef FP_INFINITE
+ #define FP_INFINITE 3
+#endif
+#ifndef FP_NORMAL
+ #define FP_NORMAL 4
+#endif
+
+
+/// Main namespace for half precision functionality.
+/// This namespace contains all the functionality provided by the library.
+namespace half_float
+{
+ class half;
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ /// Library-defined half-precision literals.
+ /// Import this namespace to enable half-precision floating point literals:
+ /// ~~~~{.cpp}
+ /// using namespace half_float::literal;
+ /// half_float::half = 4.2_h;
+ /// ~~~~
+ namespace literal
+ {
+ half operator"" _h(long double);
+ }
+#endif
+
+ /// \internal
+ /// \brief Implementation details.
+ namespace detail
+ {
+ #if HALF_ENABLE_CPP11_TYPE_TRAITS
+ /// Conditional type.
+ template<bool B,typename T,typename F> struct conditional : std::conditional<B,T,F> {};
+
+ /// Helper for tag dispatching.
+ template<bool B> struct bool_type : std::integral_constant<bool,B> {};
+ using std::true_type;
+ using std::false_type;
+
+ /// Type traits for floating point types.
+ template<typename T> struct is_float : std::is_floating_point<T> {};
+ #else
+ /// Conditional type.
+ template<bool,typename T,typename> struct conditional { typedef T type; };
+ template<typename T,typename F> struct conditional<false,T,F> { typedef F type; };
+
+ /// Helper for tag dispatching.
+ template<bool> struct bool_type {};
+ typedef bool_type<true> true_type;
+ typedef bool_type<false> false_type;
+
+ /// Type traits for floating point types.
+ template<typename> struct is_float : false_type {};
+ template<typename T> struct is_float<const T> : is_float<T> {};
+ template<typename T> struct is_float<volatile T> : is_float<T> {};
+ template<typename T> struct is_float<const volatile T> : is_float<T> {};
+ template<> struct is_float<float> : true_type {};
+ template<> struct is_float<double> : true_type {};
+ template<> struct is_float<long double> : true_type {};
+ #endif
+
+ /// Type traits for floating point bits.
+ template<typename T> struct bits { typedef unsigned char type; };
+ template<typename T> struct bits<const T> : bits<T> {};
+ template<typename T> struct bits<volatile T> : bits<T> {};
+ template<typename T> struct bits<const volatile T> : bits<T> {};
+
+ #if HALF_ENABLE_CPP11_CSTDINT
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef std::uint_least16_t uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> { typedef std::uint_least32_t type; };
+
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef std::uint_least64_t type; };
+ #else
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef unsigned short uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> : conditional<std::numeric_limits<unsigned int>::digits>=32,unsigned int,unsigned long> {};
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> : conditional<std::numeric_limits<unsigned long>::digits>=64,unsigned long,unsigned long long> {};
+ #else
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef unsigned long type; };
+ #endif
+ #endif
+
+ /// Tag type for binary construction.
+ struct binary_t {};
+
+ /// Tag for binary construction.
+ HALF_CONSTEXPR_CONST binary_t binary = binary_t();
+
+ /// Temporary half-precision expression.
+ /// This class represents a half-precision expression which just stores a single-precision value internally.
+ struct expr
+ {
+ /// Conversion constructor.
+ /// \param f single-precision value to convert
+ explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
+
+ private:
+ /// Internal expression value stored in single-precision.
+ float value_;
+ };
+
+ /// SFINAE helper for generic half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member equivalent to \a T.
+ /// \tparam T type to return
+ template<typename T,typename,typename=void,typename=void> struct enable {};
+ template<typename T> struct enable<T,half,void,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,void,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,void> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,half> { typedef T type; };
+ template<typename T> struct enable<T,half,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,expr> { typedef T type; };
+
+ /// Return type for specialized generic 2-argument half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member denoting the appropriate return type.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct result : enable<expr,T,U> {};
+ template<> struct result<half,half> { typedef half type; };
+
+ /// \name Classification helpers
+ /// \{
+
+ /// Check for infinity.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if infinity
+ /// \retval false else
+ template<typename T> bool builtin_isinf(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isinf(arg);
+ #elif defined(_MSC_VER)
+ return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
+ #else
+ return arg == std::numeric_limits<T>::infinity() || arg == -std::numeric_limits<T>::infinity();
+ #endif
+ }
+
+ /// Check for NaN.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if not a number
+ /// \retval false else
+ template<typename T> bool builtin_isnan(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isnan(arg);
+ #elif defined(_MSC_VER)
+ return ::_isnan(static_cast<double>(arg)) != 0;
+ #else
+ return arg != arg;
+ #endif
+ }
+
+ /// Check sign.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if signbit set
+ /// \retval false else
+ template<typename T> bool builtin_signbit(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::signbit(arg);
+ #else
+ return arg < T() || (arg == T() && T(1)/arg < T());
+ #endif
+ }
+
+ /// \}
+ /// \name Conversion
+ /// \{
+
+ /// Convert IEEE single-precision to half-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value single-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(float value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ uint32 bits;// = *reinterpret_cast<uint32*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(float));
+/* uint16 hbits = (bits>>16) & 0x8000;
+ bits &= 0x7FFFFFFF;
+ int exp = bits >> 23;
+ if(exp == 255)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0x7FFFFF)!=0));
+ if(exp > 142)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s;
+ if(exp > 112)
+ {
+ g = (bits>>12) & 1;
+ s = (bits&0xFFF) != 0;
+ hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF);
+ }
+ else if(exp > 101)
+ {
+ int i = 125 - exp;
+ bits = (bits&0x7FFFFF) | 0x800000;
+ g = (bits>>i) & 1;
+ s = (bits&((1L<<i)-1)) != 0;
+ hbits |= bits >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s = bits != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+*/ static const uint16 base_table[512] = {
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
+ 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
+ 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
+ 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
+ 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 };
+ static const unsigned char shift_table[512] = {
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 };
+ uint16 hbits = base_table[bits>>23] + static_cast<uint16>((bits&0x7FFFFF)>>shift_table[bits>>23]);
+ if(R == std::round_to_nearest)
+ hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00)
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((static_cast<uint32>(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits)
+ #endif
+ ;
+ else if(R == std::round_toward_zero)
+ hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23];
+ else if(R == std::round_toward_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)&
+ ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511));
+ else if(R == std::round_toward_neg_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)&
+ ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255));
+ return hbits;
+ }
+
+ /// Convert IEEE double-precision to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value double-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(double value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint64 bits;// = *reinterpret_cast<uint64*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(double));
+ uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
+ uint16 hbits = (hi>>16) & 0x8000;
+ hi &= 0x7FFFFFFF;
+ int exp = hi >> 20;
+ if(exp == 2047)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0xFFFFFFFFFFFFF)!=0));
+ if(exp > 1038)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s = lo != 0;
+ if(exp > 1008)
+ {
+ g = (hi>>9) & 1;
+ s |= (hi&0x1FF) != 0;
+ hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF);
+ }
+ else if(exp > 997)
+ {
+ int i = 1018 - exp;
+ hi = (hi&0xFFFFF) | 0x100000;
+ g = (hi>>i) & 1;
+ s |= (hi&((1L<<i)-1)) != 0;
+ hbits |= hi >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s |= hi != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+ return hbits;
+ }
+
+ /// Convert non-IEEE floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half_impl(T value, ...)
+ {
+ uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
+ if(value == T())
+ return hbits;
+ if(builtin_isnan(value))
+ return hbits | 0x7FFF;
+ if(builtin_isinf(value))
+ return hbits | 0x7C00;
+ int exp;
+ std::frexp(value, &exp);
+ if(exp > 16)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ else if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ if(exp < -13)
+ value = std::ldexp(value, 24);
+ else
+ {
+ value = std::ldexp(value, 11-exp);
+ hbits |= ((exp+13)<<10);
+ }
+ T ival, frac = std::modf(value, &ival);
+ hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
+ if(R == std::round_to_nearest)
+ {
+ frac = std::abs(frac);
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits);
+ #else
+ hbits += frac >= T(0.5);
+ #endif
+ }
+ else if(R == std::round_toward_infinity)
+ hbits += frac > T();
+ else if(R == std::round_toward_neg_infinity)
+ hbits += frac < T();
+ return hbits;
+ }
+
+ /// Convert floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half(T value)
+ {
+ return float2half_impl<R>(value, bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam S `true` if value negative, `false` else
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value non-negative integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,bool S,typename T> uint16 int2half_impl(T value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "int to half conversion only supports builtin integer types");
+ #endif
+ if(S)
+ value = -value;
+ uint16 bits = S << 15;
+ if(value > 0xFFFF)
+ {
+ if(R == std::round_toward_infinity)
+ bits |= 0x7C00 - S;
+ else if(R == std::round_toward_neg_infinity)
+ bits |= 0x7BFF + S;
+ else
+ bits |= 0x7BFF + (R!=std::round_toward_zero);
+ }
+ else if(value)
+ {
+ unsigned int m = value, exp = 24;
+ for(; m<0x400; m<<=1,--exp) ;
+ for(; m>0x7FF; m>>=1,++exp) ;
+ bits |= (exp<<10) + m;
+ if(exp > 24)
+ {
+ if(R == std::round_to_nearest)
+ bits += (value>>(exp-25)) & 1
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((1<<(exp-25))-1)&value)!=0)|bits)
+ #endif
+ ;
+ else if(R == std::round_toward_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & !S;
+ else if(R == std::round_toward_neg_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & S;
+ }
+ }
+ return bits;
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 int2half(T value)
+ {
+ return (value<0) ? int2half_impl<R,true>(value) : int2half_impl<R,false>(value);
+ }
+
+ /// Convert half-precision to IEEE single-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \param value binary representation of half-precision value
+ /// \return single-precision value
+ inline float half2float_impl(uint16 value, float, true_type)
+ {
+ typedef bits<float>::type uint32;
+/* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ bits |= 0x38000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,bits-=0x800000) ;
+ bits += static_cast<uint32>(abs) << 13;
+ }
+*/ static const uint32 mantissa_table[2048] = {
+ 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000,
+ 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000,
+ 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
+ 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000,
+ 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
+ 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
+ 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000,
+ 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000,
+ 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
+ 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
+ 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000,
+ 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
+ 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000,
+ 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000,
+ 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
+ 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000,
+ 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000,
+ 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
+ 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000,
+ 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
+ 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
+ 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000,
+ 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000,
+ 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
+ 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
+ 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000,
+ 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
+ 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000,
+ 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000,
+ 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
+ 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000,
+ 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000,
+ 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
+ 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000,
+ 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
+ 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
+ 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000,
+ 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000,
+ 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
+ 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
+ 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000,
+ 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
+ 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000,
+ 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000,
+ 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
+ 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000,
+ 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000,
+ 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
+ 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000,
+ 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
+ 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
+ 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000,
+ 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000,
+ 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
+ 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
+ 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000,
+ 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
+ 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000,
+ 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000,
+ 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
+ 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000,
+ 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000,
+ 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
+ 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000,
+ 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
+ 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
+ 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000,
+ 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000,
+ 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
+ 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
+ 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000,
+ 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
+ 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000,
+ 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000,
+ 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
+ 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000,
+ 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000,
+ 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
+ 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000,
+ 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
+ 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
+ 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000,
+ 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000,
+ 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
+ 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
+ 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000,
+ 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
+ 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000,
+ 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000,
+ 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
+ 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000,
+ 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000,
+ 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
+ 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000,
+ 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
+ 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
+ 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000,
+ 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000,
+ 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
+ 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
+ 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000,
+ 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
+ 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000,
+ 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000,
+ 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
+ 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000,
+ 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000,
+ 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
+ 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000,
+ 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
+ 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
+ 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000,
+ 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000,
+ 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
+ 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
+ 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000,
+ 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
+ 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000,
+ 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000,
+ 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
+ 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000,
+ 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000,
+ 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
+ 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000,
+ 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
+ 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
+ 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000,
+ 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 };
+ static const uint32 exponent_table[64] = {
+ 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000,
+ 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000,
+ 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
+ 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 };
+ static const unsigned short offset_table[64] = {
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 };
+ uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];
+// return *reinterpret_cast<float*>(&bits); //violating strict aliasing!
+ float out;
+ std::memcpy(&out, &bits, sizeof(float));
+ return out;
+ }
+
+ /// Convert half-precision to IEEE double-precision.
+ /// \param value binary representation of half-precision value
+ /// \return double-precision value
+ inline double half2float_impl(uint16 value, double, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint32 hi = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ hi |= 0x3F000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,hi-=0x100000) ;
+ hi += static_cast<uint32>(abs) << 10;
+ }
+ uint64 bits = static_cast<uint64>(hi) << 32;
+// return *reinterpret_cast<double*>(&bits); //violating strict aliasing!
+ double out;
+ std::memcpy(&out, &bits, sizeof(double));
+ return out;
+ }
+
+ /// Convert half-precision to non-IEEE floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float_impl(uint16 value, T, ...)
+ {
+ T out;
+ int abs = value & 0x7FFF;
+ if(abs > 0x7C00)
+ out = std::numeric_limits<T>::has_quiet_NaN ? std::numeric_limits<T>::quiet_NaN() : T();
+ else if(abs == 0x7C00)
+ out = std::numeric_limits<T>::has_infinity ? std::numeric_limits<T>::infinity() : std::numeric_limits<T>::max();
+ else if(abs > 0x3FF)
+ out = std::ldexp(static_cast<T>((abs&0x3FF)|0x400), (abs>>10)-25);
+ else
+ out = std::ldexp(static_cast<T>(abs), -24);
+ return (value&0x8000) ? -out : out;
+ }
+
+ /// Convert half-precision to floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float(uint16 value)
+ {
+ return half2float_impl(value, T(), bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,bool E,typename T> T half2int_impl(uint16 value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "half to int conversion only supports builtin integer types");
+ #endif
+ unsigned int e = value & 0x7FFF;
+ if(e >= 0x7C00)
+ return (value&0x8000) ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
+ if(e < 0x3800)
+ {
+ if(R == std::round_toward_infinity)
+ return T(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ return -T(value>0x8000);
+ return T();
+ }
+ unsigned int m = (value&0x3FF) | 0x400;
+ e >>= 10;
+ if(e < 25)
+ {
+ if(R == std::round_to_nearest)
+ m += (1<<(24-e)) - (~(m>>(25-e))&E);
+ else if(R == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(25-e))-1U);
+ else if(R == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(25-e))-1U);
+ m >>= 25 - e;
+ }
+ else
+ m <<= e - 25;
+ return (value&0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,typename T> T half2int(uint16 value) { return half2int_impl<R,HALF_ROUND_TIES_TO_EVEN,T>(value); }
+
+ /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero.
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<typename T> T half2int_up(uint16 value) { return half2int_impl<std::round_to_nearest,0,T>(value); }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R,bool E> uint16 round_half_impl(uint16 value)
+ {
+ unsigned int e = value & 0x7FFF;
+ uint16 result = value;
+ if(e < 0x3C00)
+ {
+ result &= 0x8000;
+ if(R == std::round_to_nearest)
+ result |= 0x3C00U & -(e>=(0x3800+E));
+ else if(R == std::round_toward_infinity)
+ result |= 0x3C00U & -(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ result |= 0x3C00U & -(value>0x8000);
+ }
+ else if(e < 0x6400)
+ {
+ e = 25 - (e>>10);
+ unsigned int mask = (1<<e) - 1;
+ if(R == std::round_to_nearest)
+ result += (1<<(e-1)) - (~(result>>e)&E);
+ else if(R == std::round_toward_infinity)
+ result += mask & ((value>>15)-1);
+ else if(R == std::round_toward_neg_infinity)
+ result += mask & -(value>>15);
+ result &= ~mask;
+ }
+ return result;
+ }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R> uint16 round_half(uint16 value) { return round_half_impl<R,HALF_ROUND_TIES_TO_EVEN>(value); }
+
+ /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero.
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ inline uint16 round_half_up(uint16 value) { return round_half_impl<std::round_to_nearest,0>(value); }
+ /// \}
+
+ struct functions;
+ template<typename> struct unary_specialized;
+ template<typename,typename> struct binary_specialized;
+ template<typename,typename,std::float_round_style> struct half_caster;
+ }
+
+ /// Half-precision floating point type.
+ /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and
+ /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and
+ /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations
+ /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to
+ /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic
+ /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type).
+ ///
+ /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and
+ /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which
+ /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the
+ /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of
+ /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most
+ /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit
+ /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if
+ /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on
+ /// nearly any reasonable platform.
+ ///
+ /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable
+ /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation.
+ class half
+ {
+ friend struct detail::functions;
+ friend struct detail::unary_specialized<half>;
+ friend struct detail::binary_specialized<half,half>;
+ template<typename,typename,std::float_round_style> friend struct detail::half_caster;
+ friend class std::numeric_limits<half>;
+ #if HALF_ENABLE_CPP11_HASH
+ friend struct std::hash<half>;
+ #endif
+ #if HALF_ENABLE_CPP11_USER_LITERALS
+ friend half literal::operator"" _h(long double);
+ #endif
+
+ public:
+ /// Default constructor.
+ /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics
+ /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics.
+ HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
+
+ /// Copy constructor.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ half(detail::expr rhs) : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
+
+ /// Conversion constructor.
+ /// \param rhs float to convert
+ explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ operator float() const { return detail::half2float<float>(data_); }
+
+ /// Assignment operator.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ /// \return reference to this half
+ half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to add
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator+=(T rhs) { return *this += static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to subtract
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator-=(T rhs) { return *this -= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to multiply with
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator*=(T rhs) { return *this *= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to divide by
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator/=(T rhs) { return *this /= static_cast<float>(rhs); }
+
+ /// Assignment operator.
+ /// \param rhs single-precision value to copy from
+ /// \return reference to this half
+ half& operator=(float rhs) { data_ = detail::float2half<round_style>(rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to add
+ /// \return reference to this half
+ half& operator+=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)+rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to subtract
+ /// \return reference to this half
+ half& operator-=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)-rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to multiply with
+ /// \return reference to this half
+ half& operator*=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)*rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to divide by
+ /// \return reference to this half
+ half& operator/=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)/rhs); return *this; }
+
+ /// Prefix increment.
+ /// \return incremented half value
+ half& operator++() { return *this += 1.0f; }
+
+ /// Prefix decrement.
+ /// \return decremented half value
+ half& operator--() { return *this -= 1.0f; }
+
+ /// Postfix increment.
+ /// \return non-incremented half value
+ half operator++(int) { half out(*this); ++*this; return out; }
+
+ /// Postfix decrement.
+ /// \return non-decremented half value
+ half operator--(int) { half out(*this); --*this; return out; }
+
+ private:
+ /// Rounding mode to use
+ static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE);
+
+ /// Constructor.
+ /// \param bits binary representation to set half to
+ HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {}
+
+ /// Internal binary representation
+ detail::uint16 data_;
+ };
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ namespace literal
+ {
+ /// Half literal.
+ /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due
+ /// to rather involved conversions.
+ /// \param value literal value
+ /// \return half with given value (if representable)
+ inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half<half::round_style>(value)); }
+ }
+#endif
+
+ namespace detail
+ {
+ /// Wrapper implementing unspecialized half-precision functions.
+ struct functions
+ {
+ /// Addition implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision sum stored in single-precision
+ static expr plus(float x, float y) { return expr(x+y); }
+
+ /// Subtraction implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision difference stored in single-precision
+ static expr minus(float x, float y) { return expr(x-y); }
+
+ /// Multiplication implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision product stored in single-precision
+ static expr multiplies(float x, float y) { return expr(x*y); }
+
+ /// Division implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision quotient stored in single-precision
+ static expr divides(float x, float y) { return expr(x/y); }
+
+ /// Output implementation.
+ /// \param out stream to write to
+ /// \param arg value to write
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_ostream<charT,traits>& write(std::basic_ostream<charT,traits> &out, float arg) { return out << arg; }
+
+ /// Input implementation.
+ /// \param in stream to read from
+ /// \param arg half to read into
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_istream<charT,traits>& read(std::basic_istream<charT,traits> &in, half &arg)
+ {
+ float f;
+ if(in >> f)
+ arg = f;
+ return in;
+ }
+
+ /// Modulo implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remainder(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remainder(x, y));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return expr(builtin_signbit(x) ? -0.0f : 0.0f);
+ ax = std::fmod(ax, ay+ay);
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ if(ax >= y2)
+ ax -= ay;
+ }
+ return expr(builtin_signbit(x) ? -ax : ax);
+ #endif
+ }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store quotient bits at
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remquo(float x, float y, int *quo)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remquo(x, y, quo));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ bool sign = builtin_signbit(x), qsign = static_cast<bool>(sign^builtin_signbit(y));
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
+ ax = std::fmod(ax, 8.0f*ay);
+ int cquo = 0;
+ if(ax >= 4.0f * ay)
+ {
+ ax -= 4.0f * ay;
+ cquo += 4;
+ }
+ if(ax >= 2.0f * ay)
+ {
+ ax -= 2.0f * ay;
+ cquo += 2;
+ }
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ ++cquo;
+ if(ax >= y2)
+ {
+ ax -= ay;
+ ++cquo;
+ }
+ }
+ return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
+ #endif
+ }
+
+ /// Positive difference implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Positive difference stored in single-precision
+ static expr fdim(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fdim(x, y));
+ #else
+ return expr((x<=y) ? 0.0f : (x-y));
+ #endif
+ }
+
+ /// Fused multiply-add implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return \a x * \a y + \a z stored in single-precision
+ static expr fma(float x, float y, float z)
+ {
+ #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
+ return expr(std::fma(x, y, z));
+ #else
+ return expr(x*y+z);
+ #endif
+ }
+
+ /// Get NaN.
+ /// \return Half-precision quiet NaN
+ static half nanh() { return half(binary, 0x7FFF); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp(float arg) { return expr(std::exp(arg)); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr expm1(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::expm1(arg));
+ #else
+ return expr(static_cast<float>(std::exp(static_cast<double>(arg))-1.0));
+ #endif
+ }
+
+ /// Binary exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::exp2(arg));
+ #else
+ return expr(static_cast<float>(std::exp(arg*0.69314718055994530941723212145818)));
+ #endif
+ }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log(float arg) { return expr(std::log(arg)); }
+
+ /// Common logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log10(float arg) { return expr(std::log10(arg)); }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log1p(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log1p(arg));
+ #else
+ return expr(static_cast<float>(std::log(1.0+arg)));
+ #endif
+ }
+
+ /// Binary logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log2(arg));
+ #else
+ return expr(static_cast<float>(std::log(static_cast<double>(arg))*1.4426950408889634073599246810019));
+ #endif
+ }
+
+ /// Square root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
+
+ /// Cubic root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cbrt(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::cbrt(arg));
+ #else
+ if(builtin_isnan(arg) || builtin_isinf(arg))
+ return expr(arg);
+ return expr(builtin_signbit(arg) ? -static_cast<float>(std::pow(-static_cast<double>(arg), 1.0/3.0)) :
+ static_cast<float>(std::pow(static_cast<double>(arg), 1.0/3.0)));
+ #endif
+ }
+
+ /// Hypotenuse implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr hypot(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::hypot(x, y));
+ #else
+ return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits<float>::infinity() :
+ static_cast<float>(std::sqrt(static_cast<double>(x)*x+static_cast<double>(y)*y)));
+ #endif
+ }
+
+ /// Power implementation.
+ /// \param base value to exponentiate
+ /// \param exp power to expontiate to
+ /// \return function value stored in single-preicision
+ static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
+
+ /// Sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sin(float arg) { return expr(std::sin(arg)); }
+
+ /// Cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cos(float arg) { return expr(std::cos(arg)); }
+
+ /// Tan implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tan(float arg) { return expr(std::tan(arg)); }
+
+ /// Arc sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asin(float arg) { return expr(std::asin(arg)); }
+
+ /// Arc cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acos(float arg) { return expr(std::acos(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atan(float arg) { return expr(std::atan(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
+
+ /// Hyperbolic sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sinh(float arg) { return expr(std::sinh(arg)); }
+
+ /// Hyperbolic cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cosh(float arg) { return expr(std::cosh(arg)); }
+
+ /// Hyperbolic tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tanh(float arg) { return expr(std::tanh(arg)); }
+
+ /// Hyperbolic area sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asinh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::asinh(arg));
+ #else
+ return expr((arg==-std::numeric_limits<float>::infinity()) ? arg : static_cast<float>(std::log(arg+std::sqrt(arg*arg+1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acosh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::acosh(arg));
+ #else
+ return expr((arg<-1.0f) ? std::numeric_limits<float>::quiet_NaN() : static_cast<float>(std::log(arg+std::sqrt(arg*arg-1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atanh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::atanh(arg));
+ #else
+ return expr(static_cast<float>(0.5*std::log((1.0+arg)/(1.0-arg))));
+ #endif
+ }
+
+ /// Error function implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erf(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erf(arg));
+ #else
+ return expr(static_cast<float>(erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Complementary implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erfc(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erfc(arg));
+ #else
+ return expr(static_cast<float>(1.0-erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr lgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::lgamma(arg));
+ #else
+ if(builtin_isinf(arg))
+ return expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::infinity());
+ return expr(static_cast<float>(1.1447298858494001741434273513531-
+ std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg)));
+ }
+ return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::tgamma(arg));
+ #else
+ if(arg == 0.0f)
+ return builtin_signbit(arg) ? expr(-std::numeric_limits<float>::infinity()) : expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg)));
+ return expr(static_cast<float>((std::fmod(i, 2.0f)==0.0f) ? -value : value));
+ }
+ if(builtin_isinf(arg))
+ return expr(arg);
+ return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
+ #endif
+ }
+
+ /// Floor implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half floor(half arg) { return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_)); }
+
+ /// Ceiling implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half ceil(half arg) { return half(binary, round_half<std::round_toward_infinity>(arg.data_)); }
+
+ /// Truncation implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half trunc(half arg) { return half(binary, round_half<std::round_toward_zero>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half rint(half arg) { return half(binary, round_half<half::round_style>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lrint(half arg) { return detail::half2int<half::round_style,long>(arg.data_); }
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llround(half arg) { return detail::half2int_up<long long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llrint(half arg) { return detail::half2int<half::round_style,long long>(arg.data_); }
+ #endif
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return normalized significant
+ static half frexp(half arg, int *exp)
+ {
+ int m = arg.data_ & 0x7FFF, e = -14;
+ if(m >= 0x7C00 || !m)
+ return *exp = 0, arg;
+ for(; m<0x400; m<<=1,--e) ;
+ return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF));
+ }
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+ static half modf(half arg, half *iptr)
+ {
+ unsigned int e = arg.data_ & 0x7FFF;
+ if(e >= 0x6400)
+ return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00)));
+ if(e < 0x3C00)
+ return iptr->data_ = arg.data_ & 0x8000, arg;
+ e >>= 10;
+ unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask;
+ iptr->data_ = arg.data_ & ~mask;
+ if(!m)
+ return half(binary, arg.data_&0x8000);
+ for(; m<0x400; m<<=1,--e) ;
+ return half(binary, static_cast<uint16>((arg.data_&0x8000)|(e<<10)|(m&0x3FF)));
+ }
+
+ /// Scaling implementation.
+ /// \param arg number to scale
+ /// \param exp power of two to scale by
+ /// \return scaled number
+ static half scalbln(half arg, long exp)
+ {
+ unsigned int m = arg.data_ & 0x7FFF;
+ if(m >= 0x7C00 || !m)
+ return arg;
+ for(; m<0x400; m<<=1,--exp) ;
+ exp += m >> 10;
+ uint16 value = arg.data_ & 0x8000;
+ if(exp > 30)
+ {
+ if(half::round_style == std::round_toward_zero)
+ value |= 0x7BFF;
+ else if(half::round_style == std::round_toward_infinity)
+ value |= 0x7C00 - (value>>15);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value |= 0x7BFF + (value>>15);
+ else
+ value |= 0x7C00;
+ }
+ else if(exp > 0)
+ value |= (exp<<10) | (m&0x3FF);
+ else if(exp > -11)
+ {
+ m = (m&0x3FF) | 0x400;
+ if(half::round_style == std::round_to_nearest)
+ {
+ m += 1 << -exp;
+ #if HALF_ROUND_TIES_TO_EVEN
+ m -= (m>>(1-exp)) & 1;
+ #endif
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(1-exp))-1U);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(1-exp))-1U);
+ value |= m >> (1-exp);
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ value -= (value>>15) - 1;
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value += value >> 15;
+ return half(binary, value);
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static int ilogb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return FP_ILOGB0;
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ return exp;
+ }
+ if(abs > 0x7C00)
+ return FP_ILOGBNAN;
+ return INT_MAX;
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static half logb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return half(binary, 0xFC00);
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ uint16 bits = (exp<0) << 15;
+ if(exp)
+ {
+ unsigned int m = std::abs(exp) << 6, e = 18;
+ for(; m<0x400; m<<=1,--e) ;
+ bits |= (e<<10) + m;
+ }
+ return half(binary, bits);
+ }
+ if(abs > 0x7C00)
+ return arg;
+ return half(binary, 0x7C00);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nextafter(half from, half to)
+ {
+ uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
+ if(fabs > 0x7C00)
+ return from;
+ if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs))
+ return to;
+ if(!fabs)
+ return half(binary, (to.data_&0x8000)+1);
+ bool lt = ((fabs==from.data_) ? static_cast<int>(fabs) : -static_cast<int>(fabs)) <
+ ((tabs==to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lt))<<1)-1);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nexttoward(half from, long double to)
+ {
+ if(isnan(from))
+ return from;
+ long double lfrom = static_cast<long double>(from);
+ if(builtin_isnan(to) || lfrom == to)
+ return half(static_cast<float>(to));
+ if(!(from.data_&0x7FFF))
+ return half(binary, (static_cast<detail::uint16>(builtin_signbit(to))<<15)+1);
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lfrom<to))<<1)-1);
+ }
+
+ /// Sign implementation
+ /// \param x first operand
+ /// \param y second operand
+ /// \return composed value
+ static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static int fpclassify(half arg)
+ {
+ unsigned int abs = arg.data_ & 0x7FFF;
+ return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO;
+ }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if finite number
+ /// \retval false else
+ static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if not a number
+ /// \retval false else
+ static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if normal number
+ /// \retval false else
+ static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); }
+
+ /// Sign bit implementation.
+ /// \param arg value to check
+ /// \retval true if signed
+ /// \retval false if unsigned
+ static bool signbit(half arg) { return (arg.data_&0x8000) != 0; }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x > \a y
+ /// \retval false else
+ static bool isgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x >= \a y
+ /// \retval false else
+ static bool isgreaterequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x < \a y
+ /// \retval false else
+ static bool isless(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x <= \a y
+ /// \retval false else
+ static bool islessequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either \a x > \a y nor \a x < \a y
+ /// \retval false else
+ static bool islessgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00 || yabs > 0x7C00)
+ return false;
+ int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs;
+ return a < b || a > b;
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operand unordered
+ /// \retval false else
+ static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
+
+ private:
+ static double erf(double arg)
+ {
+ if(builtin_isinf(arg))
+ return (arg<0.0) ? -1.0 : 1.0;
+ double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2)));
+ return builtin_signbit(arg) ? -value : value;
+ }
+
+ static double lgamma(double arg)
+ {
+ double v = 1.0;
+ for(; arg<8.0; ++arg) v *= arg;
+ double w = 1.0 / (arg*arg);
+ return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+
+ -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+
+ -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+
+ -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg +
+ 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg);
+ }
+ };
+
+ /// Wrapper for unary half-precision functions needing specialization for individual argument types.
+ /// \tparam T argument type
+ template<typename T> struct unary_specialized
+ {
+ /// Negation implementation.
+ /// \param arg value to negate
+ /// \return negated value
+ static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); }
+
+ /// Absolute value implementation.
+ /// \param arg function argument
+ /// \return absolute value
+ static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); }
+ };
+ template<> struct unary_specialized<expr>
+ {
+ static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
+ static expr fabs(float arg) { return expr(std::fabs(arg)); }
+ };
+
+ /// Wrapper for binary half-precision functions needing specialization for individual argument types.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct binary_specialized
+ {
+ /// Minimum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum value
+ static expr fmin(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmin(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::min(x, y));
+ #endif
+ }
+
+ /// Maximum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum value
+ static expr fmax(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmax(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::max(x, y));
+ #endif
+ }
+ };
+ template<> struct binary_specialized<half,half>
+ {
+ static half fmin(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ static half fmax(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ };
+
+ /// Helper class for half casts.
+ /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member
+ /// function and a corresponding `type` member denoting its return type.
+ /// \tparam T destination type
+ /// \tparam U source type
+ /// \tparam R rounding mode to use
+ template<typename T,typename U,std::float_round_style R=(std::float_round_style)(HALF_ROUND_STYLE)> struct half_caster {};
+ template<typename U,std::float_round_style R> struct half_caster<half,U,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<U>::value, "half_cast from non-arithmetic type unsupported");
+ #endif
+
+ static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
+
+ private:
+ static half cast_impl(U arg, true_type) { return half(binary, float2half<R>(arg)); }
+ static half cast_impl(U arg, false_type) { return half(binary, int2half<R>(arg)); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,half,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,expr,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<std::float_round_style R> struct half_caster<half,half,R>
+ {
+ static half cast(half arg) { return arg; }
+ };
+ template<std::float_round_style R> struct half_caster<half,expr,R> : half_caster<half,half,R> {};
+
+ /// \name Comparison operators
+ /// \{
+
+ /// Comparison for equality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator==(T x, U y) { return functions::isequal(x, y); }
+
+ /// Comparison for inequality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator!=(T x, U y) { return functions::isnotequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<(T x, U y) { return functions::isless(x, y); }
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>(T x, U y) { return functions::isgreater(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<=(T x, U y) { return functions::islessequal(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); }
+
+ /// \}
+ /// \name Arithmetic operators
+ /// \{
+
+ /// Add halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return sum of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator+(T x, U y) { return functions::plus(x, y); }
+
+ /// Subtract halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return difference of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator-(T x, U y) { return functions::minus(x, y); }
+
+ /// Multiply halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return product of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator*(T x, U y) { return functions::multiplies(x, y); }
+
+ /// Divide halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return quotient of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator/(T x, U y) { return functions::divides(x, y); }
+
+ /// Identity.
+ /// \param arg operand
+ /// \return uncahnged operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator+(T arg) { return arg; }
+
+ /// Negation.
+ /// \param arg operand
+ /// \return negated operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator-(T arg) { return unary_specialized<T>::negate(arg); }
+
+ /// \}
+ /// \name Input and output
+ /// \{
+
+ /// Output operator.
+ /// \param out output stream to write into
+ /// \param arg half expression to write
+ /// \return reference to output stream
+ template<typename T,typename charT,typename traits> typename enable<std::basic_ostream<charT,traits>&,T>::type
+ operator<<(std::basic_ostream<charT,traits> &out, T arg) { return functions::write(out, arg); }
+
+ /// Input operator.
+ /// \param in input stream to read from
+ /// \param arg half to read into
+ /// \return reference to input stream
+ template<typename charT,typename traits> std::basic_istream<charT,traits>&
+ operator>>(std::basic_istream<charT,traits> &in, half &arg) { return functions::read(in, arg); }
+
+ /// \}
+ /// \name Basic mathematical operations
+ /// \{
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type abs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type fabs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type fmod(T x, U y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remainder(T x, U y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store some bits of quotient at
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+
+ /// Fused multiply add.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return ( \a x * \a y ) + \a z rounded as one operation.
+// template<typename T,typename U,typename V> typename enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
+
+ /// Maximum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum of operands
+// template<typename T,typename U> typename result<T,U>::type fmax(T x, U y) { return binary_specialized<T,U>::fmax(x, y); }
+ inline half fmax(half x, half y) { return binary_specialized<half,half>::fmax(x, y); }
+ inline expr fmax(half x, expr y) { return binary_specialized<half,expr>::fmax(x, y); }
+ inline expr fmax(expr x, half y) { return binary_specialized<expr,half>::fmax(x, y); }
+ inline expr fmax(expr x, expr y) { return binary_specialized<expr,expr>::fmax(x, y); }
+
+ /// Minimum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum of operands
+// template<typename T,typename U> typename result<T,U>::type fmin(T x, U y) { return binary_specialized<T,U>::fmin(x, y); }
+ inline half fmin(half x, half y) { return binary_specialized<half,half>::fmin(x, y); }
+ inline expr fmin(half x, expr y) { return binary_specialized<half,expr>::fmin(x, y); }
+ inline expr fmin(expr x, half y) { return binary_specialized<expr,half>::fmin(x, y); }
+ inline expr fmin(expr x, expr y) { return binary_specialized<expr,expr>::fmin(x, y); }
+
+ /// Positive difference.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return \a x - \a y or 0 if difference negative
+// template<typename T,typename U> typename enable<expr,T,U>::type fdim(T x, U y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
+
+ /// Get NaN value.
+ /// \return quiet NaN
+ inline half nanh(const char*) { return functions::nanh(); }
+
+ /// \}
+ /// \name Exponential functions
+ /// \{
+
+ /// Exponential function.
+ /// \param arg function argument
+ /// \return e raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp(T arg) { return functions::exp(arg); }
+ inline expr exp(half arg) { return functions::exp(arg); }
+ inline expr exp(expr arg) { return functions::exp(arg); }
+
+ /// Exponential minus one.
+ /// \param arg function argument
+ /// \return e raised to \a arg subtracted by 1
+// template<typename T> typename enable<expr,T>::type expm1(T arg) { return functions::expm1(arg); }
+ inline expr expm1(half arg) { return functions::expm1(arg); }
+ inline expr expm1(expr arg) { return functions::expm1(arg); }
+
+ /// Binary exponential.
+ /// \param arg function argument
+ /// \return 2 raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp2(T arg) { return functions::exp2(arg); }
+ inline expr exp2(half arg) { return functions::exp2(arg); }
+ inline expr exp2(expr arg) { return functions::exp2(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base e
+// template<typename T> typename enable<expr,T>::type log(T arg) { return functions::log(arg); }
+ inline expr log(half arg) { return functions::log(arg); }
+ inline expr log(expr arg) { return functions::log(arg); }
+
+ /// Common logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 10
+// template<typename T> typename enable<expr,T>::type log10(T arg) { return functions::log10(arg); }
+ inline expr log10(half arg) { return functions::log10(arg); }
+ inline expr log10(expr arg) { return functions::log10(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg plus 1 to base e
+// template<typename T> typename enable<expr,T>::type log1p(T arg) { return functions::log1p(arg); }
+ inline expr log1p(half arg) { return functions::log1p(arg); }
+ inline expr log1p(expr arg) { return functions::log1p(arg); }
+
+ /// Binary logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 2
+// template<typename T> typename enable<expr,T>::type log2(T arg) { return functions::log2(arg); }
+ inline expr log2(half arg) { return functions::log2(arg); }
+ inline expr log2(expr arg) { return functions::log2(arg); }
+
+ /// \}
+ /// \name Power functions
+ /// \{
+
+ /// Square root.
+ /// \param arg function argument
+ /// \return square root of \a arg
+// template<typename T> typename enable<expr,T>::type sqrt(T arg) { return functions::sqrt(arg); }
+ inline expr sqrt(half arg) { return functions::sqrt(arg); }
+ inline expr sqrt(expr arg) { return functions::sqrt(arg); }
+
+ /// Cubic root.
+ /// \param arg function argument
+ /// \return cubic root of \a arg
+// template<typename T> typename enable<expr,T>::type cbrt(T arg) { return functions::cbrt(arg); }
+ inline expr cbrt(half arg) { return functions::cbrt(arg); }
+ inline expr cbrt(expr arg) { return functions::cbrt(arg); }
+
+ /// Hypotenuse function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return square root of sum of squares without internal over- or underflows
+// template<typename T,typename U> typename enable<expr,T,U>::type hypot(T x, U y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
+
+ /// Power function.
+ /// \param base first argument
+ /// \param exp second argument
+ /// \return \a base raised to \a exp
+// template<typename T,typename U> typename enable<expr,T,U>::type pow(T base, U exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
+
+ /// \}
+ /// \name Trigonometric functions
+ /// \{
+
+ /// Sine function.
+ /// \param arg function argument
+ /// \return sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sin(T arg) { return functions::sin(arg); }
+ inline expr sin(half arg) { return functions::sin(arg); }
+ inline expr sin(expr arg) { return functions::sin(arg); }
+
+ /// Cosine function.
+ /// \param arg function argument
+ /// \return cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cos(T arg) { return functions::cos(arg); }
+ inline expr cos(half arg) { return functions::cos(arg); }
+ inline expr cos(expr arg) { return functions::cos(arg); }
+
+ /// Tangent function.
+ /// \param arg function argument
+ /// \return tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tan(T arg) { return functions::tan(arg); }
+ inline expr tan(half arg) { return functions::tan(arg); }
+ inline expr tan(expr arg) { return functions::tan(arg); }
+
+ /// Arc sine.
+ /// \param arg function argument
+ /// \return arc sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asin(T arg) { return functions::asin(arg); }
+ inline expr asin(half arg) { return functions::asin(arg); }
+ inline expr asin(expr arg) { return functions::asin(arg); }
+
+ /// Arc cosine function.
+ /// \param arg function argument
+ /// \return arc cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acos(T arg) { return functions::acos(arg); }
+ inline expr acos(half arg) { return functions::acos(arg); }
+ inline expr acos(expr arg) { return functions::acos(arg); }
+
+ /// Arc tangent function.
+ /// \param arg function argument
+ /// \return arc tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atan(T arg) { return functions::atan(arg); }
+ inline expr atan(half arg) { return functions::atan(arg); }
+ inline expr atan(expr arg) { return functions::atan(arg); }
+
+ /// Arc tangent function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return arc tangent value
+// template<typename T,typename U> typename enable<expr,T,U>::type atan2(T x, U y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
+
+ /// \}
+ /// \name Hyperbolic functions
+ /// \{
+
+ /// Hyperbolic sine.
+ /// \param arg function argument
+ /// \return hyperbolic sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sinh(T arg) { return functions::sinh(arg); }
+ inline expr sinh(half arg) { return functions::sinh(arg); }
+ inline expr sinh(expr arg) { return functions::sinh(arg); }
+
+ /// Hyperbolic cosine.
+ /// \param arg function argument
+ /// \return hyperbolic cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cosh(T arg) { return functions::cosh(arg); }
+ inline expr cosh(half arg) { return functions::cosh(arg); }
+ inline expr cosh(expr arg) { return functions::cosh(arg); }
+
+ /// Hyperbolic tangent.
+ /// \param arg function argument
+ /// \return hyperbolic tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tanh(T arg) { return functions::tanh(arg); }
+ inline expr tanh(half arg) { return functions::tanh(arg); }
+ inline expr tanh(expr arg) { return functions::tanh(arg); }
+
+ /// Hyperbolic area sine.
+ /// \param arg function argument
+ /// \return area sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asinh(T arg) { return functions::asinh(arg); }
+ inline expr asinh(half arg) { return functions::asinh(arg); }
+ inline expr asinh(expr arg) { return functions::asinh(arg); }
+
+ /// Hyperbolic area cosine.
+ /// \param arg function argument
+ /// \return area cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acosh(T arg) { return functions::acosh(arg); }
+ inline expr acosh(half arg) { return functions::acosh(arg); }
+ inline expr acosh(expr arg) { return functions::acosh(arg); }
+
+ /// Hyperbolic area tangent.
+ /// \param arg function argument
+ /// \return area tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atanh(T arg) { return functions::atanh(arg); }
+ inline expr atanh(half arg) { return functions::atanh(arg); }
+ inline expr atanh(expr arg) { return functions::atanh(arg); }
+
+ /// \}
+ /// \name Error and gamma functions
+ /// \{
+
+ /// Error function.
+ /// \param arg function argument
+ /// \return error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erf(T arg) { return functions::erf(arg); }
+ inline expr erf(half arg) { return functions::erf(arg); }
+ inline expr erf(expr arg) { return functions::erf(arg); }
+
+ /// Complementary error function.
+ /// \param arg function argument
+ /// \return 1 minus error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erfc(T arg) { return functions::erfc(arg); }
+ inline expr erfc(half arg) { return functions::erfc(arg); }
+ inline expr erfc(expr arg) { return functions::erfc(arg); }
+
+ /// Natural logarithm of gamma function.
+ /// \param arg function argument
+ /// \return natural logarith of gamma function for \a arg
+// template<typename T> typename enable<expr,T>::type lgamma(T arg) { return functions::lgamma(arg); }
+ inline expr lgamma(half arg) { return functions::lgamma(arg); }
+ inline expr lgamma(expr arg) { return functions::lgamma(arg); }
+
+ /// Gamma function.
+ /// \param arg function argument
+ /// \return gamma function value of \a arg
+// template<typename T> typename enable<expr,T>::type tgamma(T arg) { return functions::tgamma(arg); }
+ inline expr tgamma(half arg) { return functions::tgamma(arg); }
+ inline expr tgamma(expr arg) { return functions::tgamma(arg); }
+
+ /// \}
+ /// \name Rounding
+ /// \{
+
+ /// Nearest integer not less than half value.
+ /// \param arg half to round
+ /// \return nearest integer not less than \a arg
+// template<typename T> typename enable<half,T>::type ceil(T arg) { return functions::ceil(arg); }
+ inline half ceil(half arg) { return functions::ceil(arg); }
+ inline half ceil(expr arg) { return functions::ceil(arg); }
+
+ /// Nearest integer not greater than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater than \a arg
+// template<typename T> typename enable<half,T>::type floor(T arg) { return functions::floor(arg); }
+ inline half floor(half arg) { return functions::floor(arg); }
+ inline half floor(expr arg) { return functions::floor(arg); }
+
+ /// Nearest integer not greater in magnitude than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater in magnitude than \a arg
+// template<typename T> typename enable<half,T>::type trunc(T arg) { return functions::trunc(arg); }
+ inline half trunc(half arg) { return functions::trunc(arg); }
+ inline half trunc(expr arg) { return functions::trunc(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<half,T>::type round(T arg) { return functions::round(arg); }
+ inline half round(half arg) { return functions::round(arg); }
+ inline half round(expr arg) { return functions::round(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long,T>::type lround(T arg) { return functions::lround(arg); }
+ inline long lround(half arg) { return functions::lround(arg); }
+ inline long lround(expr arg) { return functions::lround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type nearbyint(T arg) { return functions::nearbyint(arg); }
+ inline half nearbyint(half arg) { return functions::rint(arg); }
+ inline half nearbyint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type rint(T arg) { return functions::rint(arg); }
+ inline half rint(half arg) { return functions::rint(arg); }
+ inline half rint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long,T>::type lrint(T arg) { return functions::lrint(arg); }
+ inline long lrint(half arg) { return functions::lrint(arg); }
+ inline long lrint(expr arg) { return functions::lrint(arg); }
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long long,T>::type llround(T arg) { return functions::llround(arg); }
+ inline long long llround(half arg) { return functions::llround(arg); }
+ inline long long llround(expr arg) { return functions::llround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long long,T>::type llrint(T arg) { return functions::llrint(arg); }
+ inline long long llrint(half arg) { return functions::llrint(arg); }
+ inline long long llrint(expr arg) { return functions::llrint(arg); }
+ #endif
+
+ /// \}
+ /// \name Floating point manipulation
+ /// \{
+
+ /// Decompress floating point number.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return significant in range [0.5, 1)
+// template<typename T> typename enable<half,T>::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract integer and fractional parts.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+// template<typename T> typename enable<half,T>::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+ /// \retval FP_ILOGB0 for zero
+ /// \retval FP_ILOGBNAN for NaN
+ /// \retval MAX_INT for infinity
+// template<typename T> typename enable<int,T>::type ilogb(T arg) { return functions::ilogb(arg); }
+ inline int ilogb(half arg) { return functions::ilogb(arg); }
+ inline int ilogb(expr arg) { return functions::ilogb(arg); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+// template<typename T> typename enable<half,T>::type logb(T arg) { return functions::logb(arg); }
+ inline half logb(half arg) { return functions::logb(arg); }
+ inline half logb(expr arg) { return functions::logb(arg); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T,typename U> typename enable<half,T,U>::type nextafter(T from, U to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, expr to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T> typename enable<half,T>::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); }
+
+ /// Take sign.
+ /// \param x value to change sign for
+ /// \param y value to take sign from
+ /// \return value equal to \a x in magnitude and to \a y in sign
+// template<typename T,typename U> typename enable<half,T,U>::type copysign(T x, U y) { return functions::copysign(x, y); }
+ inline half copysign(half x, half y) { return functions::copysign(x, y); }
+ inline half copysign(half x, expr y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, half y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+
+ /// \}
+ /// \name Floating point classification
+ /// \{
+
+
+ /// Classify floating point value.
+ /// \param arg number to classify
+ /// \retval FP_ZERO for positive and negative zero
+ /// \retval FP_SUBNORMAL for subnormal numbers
+ /// \retval FP_INFINITY for positive and negative infinity
+ /// \retval FP_NAN for NaNs
+ /// \retval FP_NORMAL for all other (normal) values
+// template<typename T> typename enable<int,T>::type fpclassify(T arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(half arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
+
+ /// Check if finite number.
+ /// \param arg number to check
+ /// \retval true if neither infinity nor NaN
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isfinite(T arg) { return functions::isfinite(arg); }
+ inline bool isfinite(half arg) { return functions::isfinite(arg); }
+ inline bool isfinite(expr arg) { return functions::isfinite(arg); }
+
+ /// Check for infinity.
+ /// \param arg number to check
+ /// \retval true for positive or negative infinity
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isinf(T arg) { return functions::isinf(arg); }
+ inline bool isinf(half arg) { return functions::isinf(arg); }
+ inline bool isinf(expr arg) { return functions::isinf(arg); }
+
+ /// Check for NaN.
+ /// \param arg number to check
+ /// \retval true for NaNs
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isnan(T arg) { return functions::isnan(arg); }
+ inline bool isnan(half arg) { return functions::isnan(arg); }
+ inline bool isnan(expr arg) { return functions::isnan(arg); }
+
+ /// Check if normal number.
+ /// \param arg number to check
+ /// \retval true if normal number
+ /// \retval false if either subnormal, zero, infinity or NaN
+// template<typename T> typename enable<bool,T>::type isnormal(T arg) { return functions::isnormal(arg); }
+ inline bool isnormal(half arg) { return functions::isnormal(arg); }
+ inline bool isnormal(expr arg) { return functions::isnormal(arg); }
+
+ /// Check sign.
+ /// \param arg number to check
+ /// \retval true for negative number
+ /// \retval false for positive number
+// template<typename T> typename enable<bool,T>::type signbit(T arg) { return functions::signbit(arg); }
+ inline bool signbit(half arg) { return functions::signbit(arg); }
+ inline bool signbit(expr arg) { return functions::signbit(arg); }
+
+ /// \}
+ /// \name Comparison
+ /// \{
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreater(T x, U y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isless(T x, U y) { return functions::isless(x, y); }
+ inline bool isless(half x, half y) { return functions::isless(x, y); }
+ inline bool isless(half x, expr y) { return functions::isless(x, y); }
+ inline bool isless(expr x, half y) { return functions::isless(x, y); }
+ inline bool isless(expr x, expr y) { return functions::isless(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessequal(T x, U y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
+
+ /// Comarison for less or greater.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either less or greater
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessgreater(T x, U y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); }
+
+ /// Check if unordered.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if unordered (one or two NaN operands)
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isunordered(T x, U y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
+
+ /// \name Casting
+ /// \{
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ /// It uses the default rounding mode.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,typename U> T half_cast(U arg) { return half_caster<T,U>::cast(arg); }
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam R rounding mode to use.
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,std::float_round_style R,typename U> T half_cast(U arg) { return half_caster<T,U,R>::cast(arg); }
+ /// \}
+ }
+
+ using detail::operator==;
+ using detail::operator!=;
+ using detail::operator<;
+ using detail::operator>;
+ using detail::operator<=;
+ using detail::operator>=;
+ using detail::operator+;
+ using detail::operator-;
+ using detail::operator*;
+ using detail::operator/;
+ using detail::operator<<;
+ using detail::operator>>;
+
+ using detail::abs;
+ using detail::fabs;
+ using detail::fmod;
+ using detail::remainder;
+ using detail::remquo;
+ using detail::fma;
+ using detail::fmax;
+ using detail::fmin;
+ using detail::fdim;
+ using detail::nanh;
+ using detail::exp;
+ using detail::expm1;
+ using detail::exp2;
+ using detail::log;
+ using detail::log10;
+ using detail::log1p;
+ using detail::log2;
+ using detail::sqrt;
+ using detail::cbrt;
+ using detail::hypot;
+ using detail::pow;
+ using detail::sin;
+ using detail::cos;
+ using detail::tan;
+ using detail::asin;
+ using detail::acos;
+ using detail::atan;
+ using detail::atan2;
+ using detail::sinh;
+ using detail::cosh;
+ using detail::tanh;
+ using detail::asinh;
+ using detail::acosh;
+ using detail::atanh;
+ using detail::erf;
+ using detail::erfc;
+ using detail::lgamma;
+ using detail::tgamma;
+ using detail::ceil;
+ using detail::floor;
+ using detail::trunc;
+ using detail::round;
+ using detail::lround;
+ using detail::nearbyint;
+ using detail::rint;
+ using detail::lrint;
+#if HALF_ENABLE_CPP11_LONG_LONG
+ using detail::llround;
+ using detail::llrint;
+#endif
+ using detail::frexp;
+ using detail::ldexp;
+ using detail::modf;
+ using detail::scalbn;
+ using detail::scalbln;
+ using detail::ilogb;
+ using detail::logb;
+ using detail::nextafter;
+ using detail::nexttoward;
+ using detail::copysign;
+ using detail::fpclassify;
+ using detail::isfinite;
+ using detail::isinf;
+ using detail::isnan;
+ using detail::isnormal;
+ using detail::signbit;
+ using detail::isgreater;
+ using detail::isgreaterequal;
+ using detail::isless;
+ using detail::islessequal;
+ using detail::islessgreater;
+ using detail::isunordered;
+
+ using detail::half_cast;
+}
+
+
+/// Extensions to the C++ standard library.
+namespace std
+{
+ /// Numeric limits for half-precision floats.
+ /// Because of the underlying single-precision implementation of many operations, it inherits some properties from
+ /// `std::numeric_limits<float>`.
+ template<> class numeric_limits<half_float::half> : public numeric_limits<float>
+ {
+ public:
+ /// Supports signed values.
+ static HALF_CONSTEXPR_CONST bool is_signed = true;
+
+ /// Is not exact.
+ static HALF_CONSTEXPR_CONST bool is_exact = false;
+
+ /// Doesn't provide modulo arithmetic.
+ static HALF_CONSTEXPR_CONST bool is_modulo = false;
+
+ /// IEEE conformant.
+ static HALF_CONSTEXPR_CONST bool is_iec559 = true;
+
+ /// Supports infinity.
+ static HALF_CONSTEXPR_CONST bool has_infinity = true;
+
+ /// Supports quiet NaNs.
+ static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
+
+ /// Supports subnormal values.
+ static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
+
+ /// Rounding mode.
+ /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying
+ /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding
+ /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the
+ /// single-precision rounding mode.
+ static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits<float>::round_style==
+ half_float::half::round_style) ? half_float::half::round_style : round_indeterminate;
+
+ /// Significant digits.
+ static HALF_CONSTEXPR_CONST int digits = 11;
+
+ /// Significant decimal digits.
+ static HALF_CONSTEXPR_CONST int digits10 = 3;
+
+ /// Required decimal digits to represent all possible values.
+ static HALF_CONSTEXPR_CONST int max_digits10 = 5;
+
+ /// Number base.
+ static HALF_CONSTEXPR_CONST int radix = 2;
+
+ /// One more than smallest exponent.
+ static HALF_CONSTEXPR_CONST int min_exponent = -13;
+
+ /// Smallest normalized representable power of 10.
+ static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
+
+ /// One more than largest exponent
+ static HALF_CONSTEXPR_CONST int max_exponent = 16;
+
+ /// Largest finitely representable power of 10.
+ static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
+
+ /// Smallest positive normal value.
+ static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); }
+
+ /// Smallest finite value.
+ static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); }
+
+ /// Largest finite value.
+ static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); }
+
+ /// Difference between one and next representable value.
+ static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); }
+
+ /// Maximum rounding error.
+ static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW
+ { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); }
+
+ /// Positive infinity.
+ static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); }
+
+ /// Quiet NaN.
+ static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); }
+
+ /// Signalling NaN.
+ static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); }
+
+ /// Smallest positive subnormal value.
+ static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); }
+ };
+
+#if HALF_ENABLE_CPP11_HASH
+ /// Hash function for half-precision floats.
+ /// This is only defined if C++11 `std::hash` is supported and enabled.
+ template<> struct hash<half_float::half> //: unary_function<half_float::half,size_t>
+ {
+ /// Type of function argument.
+ typedef half_float::half argument_type;
+
+ /// Function return type.
+ typedef size_t result_type;
+
+ /// Compute hash function.
+ /// \param arg half to hash
+ /// \return hash value
+ result_type operator()(argument_type arg) const
+ { return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_)&-(arg.data_!=0x8000)); }
+ };
+#endif
+}
+
+
+#undef HALF_CONSTEXPR
+#undef HALF_CONSTEXPR_CONST
+#undef HALF_NOEXCEPT
+#undef HALF_NOTHROW
+#ifdef HALF_POP_WARNINGS
+ #pragma warning(pop)
+ #undef HALF_POP_WARNINGS
+#endif
+
+#endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index e3b8970..11001d7 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -25,7 +25,8 @@
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
+#include "half.h"
+#include "half.hpp"
#include "instructions.h"
#include "ptx_ir.h"
#include "opcodes.h"
@@ -41,13 +42,23 @@
#include "cuda_device_printf.h"
#include "../gpgpu-sim/gpu-sim.h"
#include "../gpgpu-sim/shader.h"
+#include <assert.h>
+#include <string.h>
+#include <sstream>
+#include <stdio.h>
+#include <string>
+#include <map>
+#include <stdlib.h>
//Jin: include device runtime for CDP
#include "cuda_device_runtime.h"
#include <stdarg.h>
+using half_float::half;
unsigned ptx_instruction::g_num_ptx_inst_uid=0;
+bool debug_tensorcore = 0;
+
const char *g_opcode_string[NUM_OPCODES] = {
#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR,
@@ -56,6 +67,104 @@ const char *g_opcode_string[NUM_OPCODES] = {
#undef OP_DEF
#undef OP_W_DEF
};
+//Using profiled information::check the TensorCoreMatrixArrangement.xls for details
+unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){
+
+ unsigned offset;
+ unsigned load_a_row[8]={0,128,0,128,64,192,64,192};
+ unsigned load_a_col[8]={0,8,0,8,4,12,4,12};
+ unsigned load_b_row[8]={0,8,0,8,4,12,4,12};
+ unsigned load_b_col[8]={0,128,0,128,64,192,64,192};
+ unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200};
+ unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140};
+ unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200};
+ unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140};
+ unsigned thread_group = thread/4;
+ unsigned in_tg_index = thread%4;
+
+ switch(wmma_type){
+ case LOAD_A:
+ if(wmma_layout==ROW)
+ offset=load_a_row[thread_group]+16*in_tg_index;
+ else
+ offset=load_a_col[thread_group]+16*in_tg_index;
+ break;
+
+
+ case LOAD_B:
+ if(wmma_layout==ROW)
+ offset=load_b_row[thread_group]+16*in_tg_index;
+ else
+ offset=load_b_col[thread_group]+16*in_tg_index;
+ break;
+
+ case LOAD_C:
+ case STORE_D:
+ if(type==F16_TYPE){
+ if(wmma_layout==ROW)
+ offset=load_c_half_row[thread_group]+16*in_tg_index;
+ else
+ offset=load_c_half_col[thread_group]+in_tg_index;
+ }
+ else{
+ if(wmma_layout==ROW)
+ offset=load_c_float_row[thread_group];
+ else
+ offset=load_c_float_col[thread_group];
+
+ switch(in_tg_index){
+ case 0:
+ break;
+ case 1:
+ if(wmma_layout==ROW)
+ offset+=16;
+ else
+ offset+=1;
+ break;
+ case 2:
+ if(wmma_layout==ROW)
+ offset+=2;
+ else
+ offset+=32;
+ break;
+ case 3:
+ if(wmma_layout==ROW)
+ offset+=18;
+ else
+ offset+=33;
+ break;
+ default:
+ abort();
+ }
+ }
+ break;
+
+ default:
+ abort();
+
+ }
+ offset = (offset/16)*stride+offset%16;
+ return offset;
+}
+
+int acc_float_offset(int index,int wmma_layout,int stride){
+
+ int c_row_offset[]={0,1,32,33,4,5,36,37};
+ int c_col_offset[]={0,16,2,18,64,80,66,82};
+ int offset;
+
+
+ if(wmma_layout==ROW)
+ offset=c_row_offset[index];
+ else if(wmma_layout==COL)
+ offset=c_col_offset[index];
+ else{
+ printf("wrong layout");
+ abort();
+ }
+ offset = (offset/16)*stride+offset%16;
+ return offset;
+}
void inst_not_implemented( const ptx_instruction * pI ) ;
ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread);
@@ -74,6 +183,64 @@ void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value )
m_last_set_operand_value = value;
}
+void ptx_thread_info::print_reg_thread(char * fname)
+{
+
+ FILE *fp= fopen(fname,"w");
+ assert(fp!=NULL);
+
+ int size = m_regs.size();
+
+ if(size>0)
+ {
+ reg_map_t reg = m_regs.back();
+
+ reg_map_t::const_iterator it;
+ for (it = reg.begin(); it != reg.end(); ++it)
+ {
+ const std::string &name = it->first->name();
+ const std::string &dec= it->first->decl_location();
+ unsigned size = it->first->get_size_in_bytes();
+ fprintf(fp,"%s %llu %s %d\n",name.c_str(),it->second, dec.c_str(),size );
+
+ }
+ //m_regs.pop_back();
+ }
+ fclose(fp);
+
+ }
+
+void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab)
+{
+
+
+ FILE * fp2 = fopen(fname, "r");
+ assert(fp2!=NULL);
+ //m_regs.push_back( reg_map_t() );
+ char line [ 200 ];
+ while ( fgets ( line, sizeof line, fp2 ) != NULL )
+ {
+ symbol *reg;
+ char * pch;
+ unsigned size;
+ pch = strtok (line," ");
+ char * name =pch;
+ reg= symtab->lookup(name);
+ ptx_reg_t data;
+ pch = strtok (NULL," ");
+ data = atoi(pch);
+ pch = strtok (NULL," ");
+ char * decl= pch;
+ pch = strtok (NULL," ");
+ size = atoi(pch);
+
+
+ m_regs.back()[reg] = data;
+ }
+ fclose ( fp2 );
+}
+
+
ptx_reg_t ptx_thread_info::get_reg( const symbol *reg )
{
static bool unfound_register_warned = false;
@@ -136,10 +303,12 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
result.u64 = sym->get_address() + op.get_addr_offset();
} else if ( op.is_shared() ) {
result.u64 = op.get_symbol()->get_address() + op.get_addr_offset();
+ } else if ( op.is_sstarr() ) {
+ result.u64 = op.get_symbol()->get_address() + op.get_addr_offset();
} else {
- const char *name = op.name().c_str();
- printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name );
- abort();
+ const char *name = op.name().c_str();
+ printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name );
+ abort();
}
} else if ( op.is_literal() ) {
@@ -148,6 +317,8 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
result.u64 = op.get_symbol()->get_address();
} else if ( op.is_shared() ) {
result.u64 = op.get_symbol()->get_address();
+ } else if ( op.is_sstarr() ) {
+ result.u64 = op.get_symbol()->get_address();
} else if ( op.is_const() ) {
result.u64 = op.get_symbol()->get_address();
} else if ( op.is_global() ) {
@@ -155,11 +326,21 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
} else if ( op.is_local() ) {
result.u64 = op.get_symbol()->get_address();
} else if ( op.is_function_address() ) {
- result.u64 = (size_t)op.get_symbol()->get_pc();
- } else {
+ result.u64 = (size_t)op.get_symbol()->get_pc();
+ } else if ( op.is_param_kernel()) {
+ result.u64 = op.get_symbol()->get_address();
+ }else {
const char *name = op.name().c_str();
- printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name );
- assert(0);
+ const symbol *sym2 = op.get_symbol();
+ const type_info *type2 = sym2->type();
+ const type_info_key &info2 = type2->get_key();
+ if ( info2.is_param_kernel() ) {
+ result.u64 = sym2->get_address()+ op.get_addr_offset();
+ }
+ else{
+ printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name );
+ assert(0);
+ }
}
if(op.get_operand_lohi() == 1)
@@ -335,7 +516,7 @@ unsigned get_operand_nbits( const operand_info &op )
void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements )
{
assert( op.is_vector() );
- assert( num_elements <= 4 ); // max 4 elements in a vector
+ assert( num_elements <= 8 );
for (int idx = num_elements - 1; idx >= 0; --idx) {
const symbol *sym = NULL;
@@ -640,6 +821,33 @@ void ptx_thread_info::set_vector_operand_values( const operand_info &dst,
m_last_set_operand_value = data1;
}
+void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst,
+ const ptx_reg_t &data1,
+ const ptx_reg_t &data2,
+ const ptx_reg_t &data3,
+ const ptx_reg_t &data4,
+ const ptx_reg_t &data5,
+ const ptx_reg_t &data6,
+ const ptx_reg_t &data7,
+ const ptx_reg_t &data8 )
+{
+ unsigned num_elements = dst.get_vect_nelem();
+ if (num_elements == 8) {
+ set_reg(dst.vec_symbol(0), data1);
+ set_reg(dst.vec_symbol(1), data2);
+ set_reg(dst.vec_symbol(2), data3);
+ set_reg(dst.vec_symbol(3), data4);
+ set_reg(dst.vec_symbol(4), data5);
+ set_reg(dst.vec_symbol(5), data6);
+ set_reg(dst.vec_symbol(6), data7);
+ set_reg(dst.vec_symbol(7), data8);
+ }
+ else{
+ printf("error:set_wmma_vector_operands");
+ }
+
+ m_last_set_operand_value = data8;
+}
#define my_abs(a) (((a)<0)?(-a):(a))
@@ -745,7 +953,7 @@ void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread )
case U64_TYPE:
data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4);
break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break;
case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break;
case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break;
default: assert(0); break;
@@ -812,7 +1020,7 @@ void add_impl( const ptx_instruction *pI, ptx_thread_info *thread )
case U64_TYPE:
data.u64 = src1_data.u64 + src2_data.u64;
break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break;
case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break;
case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break;
default: assert(0); break;
@@ -1352,45 +1560,45 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread )
const operand_info &src1 = pI->src1();
const operand_info &src2 = pI->src2();
const operand_info &src3 = pI->src3();
- ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1);
+ ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1);
ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1);
ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1);
+ ptx_reg_t data;
unsigned pos = b.u32 & 0xFF;
unsigned len = c.u32 & 0xFF;
- unsigned d = 0;
switch (i_type)
{
case U32_TYPE:
{
unsigned mask;
- d = a.u32 >> pos;
+ data.u32 = src.u32 >> pos;
mask = 0xFFFFFFFF >> (32 - len);
- d &= mask;
+ data.u32 &= mask;
break;
}
case U64_TYPE:
{
unsigned long mask;
- d = a.u64 >> pos;
+ data.u64 = src.u64 >> pos;
mask = 0xFFFFFFFFFFFFFFFF >> (64 - len);
- d &= mask;
+ data.u64 &= mask;
break;
}
case S32_TYPE:
{
unsigned mask;
unsigned min = MY_MIN_I(pos + len - 1, msb);
- unsigned sbit = len == 0 ? 0 : (a.s32 >> min) & 0x1;
- d = a.s32 >> pos;
+ unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1;
+ data.s32 = src.s32 >> pos;
if (sbit > 0)
{
mask = 0xFFFFFFFF << len;
- d |= mask;
+ data.s32 |= mask;
}
else
{
mask = 0xFFFFFFFF >> (32 - len);
- d &= mask;
+ data.s32 &= mask;
}
break;
}
@@ -1398,17 +1606,17 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread )
{
unsigned long mask;
unsigned min = MY_MIN_I(pos + len - 1, msb);
- unsigned sbit = len == 0 ? 0 : (a.s64 >> min) & 0x1;
- d = a.s64 >> pos;
+ unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1;
+ data.s64 = src.s64 >> pos;
if (sbit > 0)
{
mask = 0xFFFFFFFFFFFFFFFF << len;
- d |= mask;
+ data.s64 |= mask;
}
else
{
mask = 0xFFFFFFFFFFFFFFFF >> (64 - len);
- d &= mask;
+ data.s64 &= mask;
}
break;
}
@@ -1417,10 +1625,47 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread )
abort();
return;
}
- thread->set_operand_value(dst,d, i_type, thread, pI);
+ thread->set_operand_value(dst, data, i_type, thread, pI);
}
-void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
+void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) {
+ int i,max;
+ ptx_reg_t src1_data, src2_data;
+ ptx_reg_t src3_data, src4_data, data;
+
+ const operand_info &dst = pI->dst(); //get operand info of sources and destination
+ const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register'
+ const operand_info &src2 = pI->src2();
+ const operand_info &src3 = pI->src3();
+ const operand_info &src4 = pI->src4();
+
+ unsigned i_type = pI->get_type();
+ src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1);
+ src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1);
+ src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1);
+ src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1);
+
+ switch ( i_type ) {
+ case B32_TYPE:
+ max = 32;
+ break;
+ case B64_TYPE:
+ max = 64;
+ break;
+ default:
+ printf("Execution error: type mismatch with instruction\n");
+ assert(0);
+ break;
+ }
+ data=src2_data;
+ unsigned pos = src3_data.u32 & 0xFF;
+ unsigned len = src4_data.u32 & 0xFF;
+ for(i=0;i<len && pos+i<max;i++){
+ data.u32=(~((0x00000001)<<(pos+i)))&data.u32;
+ data.u32=data.u32|((src1_data.u32&((0x00000001)<<(i)))<<(pos));
+ }
+ thread->set_operand_value(dst, data, i_type, thread, pI);
+}
void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread )
@@ -1457,9 +1702,349 @@ void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread )
assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated
}
-void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
+void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread )
+{
+ ptx_reg_t src1_data, data;
+ const operand_info &dst = pI->dst();
+ const operand_info &src1 = pI->src1();
+ unsigned i_type = pI->get_type();
+ src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1);
+
+ unsigned msb;
+ switch(i_type){
+ case B32_TYPE:
+ msb = 31;
+ for (unsigned i=0; i<=msb; i++) {
+ if((src1_data.u32 & (1 << i)))
+ data.u32 |= 1 << (msb - i);
+ }
+ break;
+ case B64_TYPE:
+ msb = 63;
+ for (unsigned i=0; i<=msb; i++) {
+ if((src1_data.u64 & (1 << i)))
+ data.u64 |= 1 << (msb - i);
+ }
+ break;
+ default: assert(0);
+ }
+ thread->set_operand_value(dst,data, i_type, thread, pI);
+}
void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
+unsigned trunc(unsigned num, unsigned precision) {
+ int mask = 1, latest_one = -1;
+ unsigned data = num;
+ for (unsigned j = 0; j < sizeof(unsigned)*8; j++) {
+ int bit = data & mask;
+ if (bit == 1) latest_one = j;
+ data >>= 1;
+ }
+ if (latest_one >= precision) {
+ // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0
+ //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision);
+ //unsigned shifted_output = num >> (latest_one-precision+1);
+ // if shifted_output is a number like 1111, don't round up
+ //if (shifted_output == (pow(2,precision)-1)) round_up = 0;
+ //num = shifted_output + round_up;
+ num >>= (latest_one-precision+1);
+ }
+ return num;
+}
+void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){
+ int offset;
+ int c_row_offset[]={0,8,0,8,4,12,4,12};
+ int c_col_offset[]={0,0,8,8,0,0,8,8};
+ int c_tg_inside_row_offset[]={0,1,0,1};
+ int c_tg_inside_col_offset[]={0,0,2,2};
+ int c_inside_row_offset[]={0,0,2,2,0,0,2,2};
+ int c_inside_col_offset[]={0,1,0,1,4,5,4,5};
+
+ offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride);
+
+ if(wmma_type==LOAD_A){
+ if(wmma_layout==ROW){
+ offset+=index+8*((thread%16)/8);
+ }
+ else{
+ offset+=64*(index/4)+index%4+128*((thread%16)/8);
+ }
+ offset=(offset/16)*stride+offset%16;
+ assg_offset=index+8*((thread%16)/8);
+ }
+ else if(wmma_type==LOAD_B){
+ if(wmma_layout==ROW){
+ offset+=64*(index/4)+index%4+128*((thread%16)/8);
+ }
+ else{
+ offset+=index+8*((thread%16)/8);
+ }
+ offset=(offset/16)*stride+offset%16;
+ assg_offset=index+8*((thread%16)/8);
+ }
+ else if( wmma_type==LOAD_C){
+ if(type==F16_TYPE){
+ row=c_row_offset[thread/4]+thread%4;
+ col=c_col_offset[thread/4]+index;
+ }
+ else{
+ row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index];
+ col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index];
+ }
+ assg_offset=index;
+ }
+
+ if(wmma_type==LOAD_A||wmma_type==LOAD_B){
+ if(wmma_layout==ROW){
+ row=offset/16;
+ col=offset%16;
+ }
+ else{
+ col=offset/16;
+ row=offset%16;
+ }
+ }
+}
+
+void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
+{
+ int i,j,k,thrd;
+ int row,col,offset;
+ ptx_reg_t matrix_a[16][16];
+ ptx_reg_t matrix_b[16][16];
+ ptx_reg_t matrix_c[16][16];
+ ptx_reg_t matrix_d[16][16];
+ ptx_reg_t src_data;
+ ptx_thread_info *thread;
+ int stride;
+
+ unsigned wmma_type = pI->get_wmma_type();
+ unsigned a_layout = pI->get_wmma_layout(0);
+ unsigned b_layout = pI->get_wmma_layout(1);
+ unsigned type = pI->get_type();
+ unsigned type2 = pI->get_type2();
+ int tid ;
+ const operand_info &dst = pI->operand_lookup(0);
+
+ if(core->get_gpu()->is_functional_sim())
+ tid= inst.warp_id_func()*core->get_warp_size();
+ else
+ tid= inst.warp_id()*core->get_warp_size();
+ unsigned thread_group_index;
+ float temp;
+ half temp2;
+
+ for (thrd=0; thrd < core->get_warp_size(); thrd++){
+ thread = core->get_thread_info()[tid+thrd];
+ if(debug_tensorcore)
+ printf("THREAD=%d\n:",thrd);
+ for(int operand_num=1;operand_num<=3;operand_num++){
+ const operand_info &src_a= pI->operand_lookup(operand_num);
+ unsigned nelem = src_a.get_vect_nelem();
+ ptx_reg_t v[8];
+ thread->get_vector_operand_values( src_a, v, nelem );
+ if(debug_tensorcore){
+ printf("Thread%d_Iteration=%d\n:",thrd,operand_num);
+ for(k=0;k<nelem;k++){
+ printf("%x ",v[k].u64);
+ }
+ printf("\n");
+ }
+ ptx_reg_t nw_v[16];
+ int hex_val;
+
+ if(!((operand_num==3)&&(type2==F32_TYPE))){
+ for(k=0;k<2*nelem;k++){
+ if(k%2==1)
+ hex_val=(v[k/2].s64&0xffff);
+ else
+ hex_val=((v[k/2].s64&0xffff0000)>>16);
+ nw_v[k].f16 =*((half *)&hex_val);
+ }
+ }
+ if(!((operand_num==3)&&(type2==F32_TYPE))){
+ for(k=0;k<2*nelem;k++){
+ temp=nw_v[k].f16;
+ if(debug_tensorcore)
+ printf("%.2f ",temp);
+ }
+ if(debug_tensorcore)
+ printf("\n");
+ }
+ else{
+ if(debug_tensorcore){
+ for(k=0;k<8;k++){
+ printf("%.2f ",v[k].f32);
+ }
+ printf("\n");
+ }
+ }
+ switch(operand_num) {
+ case 1 ://operand 1
+ for(k=0;k<8;k++){
+ mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset);
+ if(debug_tensorcore)
+ printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset);
+ matrix_a[row][col]=nw_v[offset];
+ }
+ break;
+ case 2 ://operand 2
+ for(k=0;k<8;k++){
+ mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset);
+ if(debug_tensorcore)
+ printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset);
+ matrix_b[row][col]=nw_v[offset];
+ }
+ break;
+ case 3 ://operand 3
+ for(k=0;k<8;k++){
+ mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset);
+ if(debug_tensorcore)
+ printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset);
+ if(type2!=F16_TYPE){
+ matrix_c[row][col]=v[offset];
+ }
+ else {
+ matrix_c[row][col]=nw_v[offset];
+ }
+ }
+ break;
+ default :
+ printf("Invalid Operand Index\n" );
+ }
+ }
+ if(debug_tensorcore)
+ printf("\n");
+ }
+ if(debug_tensorcore){
+ printf("MATRIX_A\n");
+ for (i=0;i<16;i++){
+ for(j=0;j<16;j++){
+ temp=matrix_a[i][j].f16;
+ printf("%.2f ",temp);
+ }
+ printf("\n");
+ }
+ printf("MATRIX_B\n");
+ for (i=0;i<16;i++){
+ for(j=0;j<16;j++){
+ temp=matrix_b[i][j].f16;
+ printf("%.2f ",temp);
+ }
+ printf("\n");
+ }
+ printf("MATRIX_C\n");
+ for (i=0;i<16;i++){
+ for(j=0;j<16;j++){
+ if(type2==F16_TYPE){
+ temp=matrix_c[i][j].f16;
+ printf("%.2f ",temp);
+ }
+ else
+ printf("%.2f ",matrix_c[i][j].f32);
+ }
+ printf("\n");
+ }
+ }
+ for (i=0;i<16;i++){
+ for(j=0;j<16;j++){
+ matrix_d[i][j].f16=0;
+ }
+ }
+
+ for (i=0;i<16;i++){
+ for(j=0;j<16;j++){
+ for(k=0;k<16;k++){
+ matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16;
+ }
+ if((type==F16_TYPE)&&(type2==F16_TYPE))
+ matrix_d[i][j].f16+=matrix_c[i][j].f16;
+ else if((type==F32_TYPE)&&(type2==F16_TYPE)){
+ temp2=matrix_d[i][j].f16+matrix_c[i][j].f16;
+ temp=temp2;
+ matrix_d[i][j].f32=temp;
+ }
+ else if((type==F16_TYPE)&&(type2==F32_TYPE)){
+ temp=matrix_d[i][j].f16;
+ temp+=matrix_c[i][j].f32;
+ matrix_d[i][j].f16=half(temp);
+ }
+ else{
+ temp=matrix_d[i][j].f16;
+ temp+=matrix_c[i][j].f32;
+ matrix_d[i][j].f32=temp;
+ }
+ }
+ }
+ if(debug_tensorcore){
+ printf("MATRIX_D\n");
+ for (i=0;i<16;i++){
+ for(j=0;j<16;j++){
+ if(type==F16_TYPE){
+ temp=matrix_d[i][j].f16;
+ printf("%.2f ",temp);
+ }
+ else
+ printf("%.2f ",matrix_d[i][j].f32);
+ }
+ printf("\n");
+ }
+ }
+ for (thrd=0; thrd < core->get_warp_size(); thrd++){
+ int row_t[8];
+ int col_t[8];
+ for(k=0;k<8;k++){
+ mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset);
+ if(debug_tensorcore)
+ printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]);
+ }
+ thread = core->get_thread_info()[tid+thrd];
+
+
+ if(type==F32_TYPE){
+ thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]);
+
+ if(debug_tensorcore)
+ {
+ printf("thread%d:",thrd);
+ for(k=0;k<8;k++){
+ printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32);
+ }
+ printf("\n");
+ }
+ }
+ else if(type==F16_TYPE){
+ if(debug_tensorcore){
+ printf("thread%d:",thrd);
+ for(k=0;k<8;k++){
+ temp=matrix_d[row_t[k]][col_t[k]].f16;
+ printf("%.2f ",temp);
+ }
+ printf("\n");
+
+ printf("thread%d:",thrd);
+ for(k=0;k<8;k++){
+ printf("%x ",matrix_d[row_t[k]][col_t[k]].f16);
+ }
+ printf("\n");
+ }
+ ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4;
+ nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16);
+ nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16);
+ nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16);
+ nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16);
+ thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4);
+ if(debug_tensorcore)
+ printf("thread%d=%x,%x,%x,%x",thrd,nw_data1.s64,nw_data2.s64,nw_data3.s64,nw_data4.s64);
+
+ }
+ else{
+ printf("wmma:mma:wrong type\n");
+ abort();
+ }
+ }
+}
+
void call_impl( const ptx_instruction *pI, ptx_thread_info *thread )
{
static unsigned call_uid_next = 1;
@@ -1467,7 +2052,19 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread )
const operand_info &target = pI->func_addr();
assert( target.is_function_address() );
const symbol *func_addr = target.get_symbol();
- const function_info *target_func = func_addr->get_pc();
+ function_info *target_func = func_addr->get_pc();
+ if (target_func->is_pdom_set()) {
+ printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() );
+ } else {
+ printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() );
+ /*
+ * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call.
+ * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash.
+ */
+ if (target_func->get_function_size() >0)
+ target_func->do_pdom();
+ target_func->set_pdom();
+ }
// check that number of args and return match function requirements
if( pI->has_return() ^ target_func->has_return() ) {
@@ -1698,7 +2295,10 @@ unsigned int saturatei(unsigned int a, unsigned int max)
ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode )
{
- assert( from_width == 32);
+ half mytemp;
+ float myfloat;
+ half_float::half tmp_h;
+ //assert( from_width == 32);
enum cuda_math::cudaRoundMode mode = cuda_math::cudaRoundZero;
switch (rounding_mode) {
@@ -1740,8 +2340,12 @@ ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign,
}
} else {
switch ( to_width ) {
- case 16: assert(0); break;
- case 32: assert(0); break; // handled by f2f
+ case 16:
+ y.f16 =half_float::half_cast<half,std::numeric_limits<float>::round_style>(x.f32);//mytemp;
+ break;
+ case 32:
+ y.f32=float(x.f16);
+ break; // handled by f2f
case 64:
y.f64 = x.f32;
break;
@@ -1891,48 +2495,53 @@ ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign,
ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode )
{
ptx_reg_t y;
- switch ( rounding_mode ) {
- case RZI_OPTION:
- y.f32 = truncf(x.f32);
- break;
- case RNI_OPTION:
-#if CUDART_VERSION >= 3000
- y.f32 = nearbyintf(x.f32);
-#else
- y.f32 = cuda_math::__internal_nearbyintf(x.f32);
-#endif
- break;
- case RMI_OPTION:
- if ((x.u32 & 0x7f800000) == 0) {
- y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign
- } else {
- y.f32 = floorf(x.f32);
- }
- break;
- case RPI_OPTION:
- if ((x.u32 & 0x7f800000) == 0) {
- y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign
- } else {
- y.f32 = ceilf(x.f32);
- }
- break;
- default:
- if ((x.u32 & 0x7f800000) == 0) {
- y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign
- } else {
- y.f32 = x.f32;
- }
- break;
- }
-#if CUDART_VERSION >= 3000
- if (isnanf(y.f32))
-#else
- if (cuda_math::__cuda___isnanf(y.f32))
-#endif
- {
- y.u32 = 0x7fffffff;
- } else if (saturation_mode) {
- y.f32 = cuda_math::__saturatef(y.f32);
+ if (from_width == 16){
+ half_float::detail::uint16 val = x.u16;
+ y.f32 = half_float::detail::half2float<float>(val);
+ }else{
+ switch ( rounding_mode ) {
+ case RZI_OPTION:
+ y.f32 = truncf(x.f32);
+ break;
+ case RNI_OPTION:
+ #if CUDART_VERSION >= 3000
+ y.f32 = nearbyintf(x.f32);
+ #else
+ y.f32 = cuda_math::__internal_nearbyintf(x.f32);
+ #endif
+ break;
+ case RMI_OPTION:
+ if ((x.u32 & 0x7f800000) == 0) {
+ y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign
+ } else {
+ y.f32 = floorf(x.f32);
+ }
+ break;
+ case RPI_OPTION:
+ if ((x.u32 & 0x7f800000) == 0) {
+ y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign
+ } else {
+ y.f32 = ceilf(x.f32);
+ }
+ break;
+ default:
+ if ((x.u32 & 0x7f800000) == 0) {
+ y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign
+ } else {
+ y.f32 = x.f32;
+ }
+ break;
+ }
+ #if CUDART_VERSION >= 3000
+ if (isnanf(y.f32))
+ #else
+ if (cuda_math::__cuda___isnanf(y.f32))
+ #endif
+ {
+ y.u32 = 0x7fffffff;
+ } else if (saturation_mode) {
+ y.f32 = cuda_math::__saturatef(y.f32);
+ }
}
return y;
@@ -1980,7 +2589,7 @@ ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_wid
{ chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f},
{ chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f},
{ chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f},
- { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2x, f2x},
+ { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x},
{ f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x},
{ d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d}
};
@@ -2002,7 +2611,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type)
case U32_TYPE:
case U64_TYPE:
printf("Trying to round an integer??\n"); assert(0); break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break;
case F32_TYPE:
data.f32 = truncf(data.f32);
break;
@@ -2025,7 +2634,13 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type)
case U32_TYPE:
case U64_TYPE:
printf("Trying to round an integer??\n"); assert(0); break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE:// assert(0); break;
+#if CUDART_VERSION >= 3000
+ data.f16 = nearbyintf(data.f16);
+#else
+ data.f16 = cuda_math::__cuda_nearbyintf(data.f16);
+#endif
+ break;
case F32_TYPE:
#if CUDART_VERSION >= 3000
data.f32 = nearbyintf(data.f32);
@@ -2048,7 +2663,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type)
case U32_TYPE:
case U64_TYPE:
printf("Trying to round an integer??\n"); assert(0); break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break;
case F32_TYPE:
data.f32 = floorf(data.f32);
break;
@@ -2067,7 +2682,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type)
case U32_TYPE:
case U64_TYPE:
printf("Trying to round an integer??\n"); assert(0); break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break;
case F32_TYPE: data.f32 = ceilf(data.f32); break;
case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break;
default: assert(0); break;
@@ -2108,7 +2723,10 @@ void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type)
case U32_TYPE:
case U64_TYPE:
printf("Trying to clamp an integer to 1??\n"); assert(0); break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: //assert(0); break;
+ if (data.f16 > 1.0f) data.f16 = 1.0f; //negative
+ if (data.f16 < 0.0f) data.f16 = 0.0f; //positive
+ break;
case F32_TYPE:
if (data.f32 > 1.0f) data.f32 = 1.0f; //negative
if (data.f32 < 0.0f) data.f32 = 0.0f; //positive
@@ -2132,8 +2750,8 @@ void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread )
unsigned rounding_mode = pI->rounding_mode();
unsigned saturation_mode = pI->saturation_mode();
- if ( to_type == F16_TYPE || from_type == F16_TYPE )
- abort();
+// if ( to_type == F16_TYPE || from_type == F16_TYPE )
+// abort();
int to_sign, from_sign;
size_t from_width, to_width;
@@ -2268,7 +2886,7 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread )
data.u32 = src1_data.u32 / src2_data.u32; break;
case B64_TYPE:
data.u64 = src1_data.u64 / src2_data.u64; break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break;
case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break;
case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break;
default: assert(0); break;
@@ -2276,6 +2894,13 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread )
thread->set_operand_value(dst,data, i_type, thread,pI);
}
+void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread )
+{
+ printf("DP4A instruction not implemented yet");
+ assert(0);
+
+}
+
void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread )
{
ptx_reg_t src1_data, src2_data, data;
@@ -2355,7 +2980,12 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand
space = param_space_kernel;
else if( ti.is_param_local() ) {
space = param_space_local;
- } else {
+ }
+ //mov r1, param-label
+ else if (ti.is_reg() ){
+ space = param_space_kernel;
+ }
+ else {
printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() );
abort();
}
@@ -2371,6 +3001,7 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand
case surf_space: mem = thread->get_surf_memory(); break;
case param_space_kernel: mem = thread->get_param_memory(); break;
case shared_space: mem = thread->m_shared_mem; break;
+ case sstarr_space: mem = thread->m_sstarr_mem; break;
case const_space: mem = thread->get_global_memory(); break;
case generic_space:
if( thread->get_ptx_version().ver() >= 2.0 ) {
@@ -2446,6 +3077,332 @@ void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread )
ld_exec(pI,thread);
}
+void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst )
+{
+ size_t size;
+ unsigned smid;
+ int t;
+ int thrd,odd,inx,k;
+ ptx_thread_info *thread;
+
+ const operand_info &src = pI->operand_lookup(1);
+ const operand_info &src1 = pI->operand_lookup(0);
+ const operand_info &src2 = pI->operand_lookup(2);
+ int tid ;
+ unsigned type = pI->get_type();
+ unsigned wmma_type = pI->get_wmma_type();
+ unsigned wmma_layout = pI->get_wmma_layout(0);
+ int stride;
+
+ if(core->get_gpu()->is_functional_sim())
+ tid= inst.warp_id_func()*core->get_warp_size();
+ else
+ tid= inst.warp_id()*core->get_warp_size();
+
+ _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store;
+ for (thrd=0; thrd < core->get_warp_size(); thrd++) {
+ thread = core->get_thread_info()[tid+thrd];
+ odd=thrd%2;
+ inx=thrd/2;
+ ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1);
+ ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1);
+ const operand_info &src_a= pI->operand_lookup(1);
+ unsigned nelem = src_a.get_vect_nelem();
+ ptx_reg_t* v= new ptx_reg_t[8];
+ thread->get_vector_operand_values( src_a, v, nelem );
+ stride=src2_data.u32;
+
+ memory_space_t space = pI->get_space();
+
+ memory_space *mem = NULL;
+ addr_t addr = addr_reg.u32;
+
+ new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD];
+ int num_mem_txn=0;
+
+ smid = thread->get_hw_sid();
+ if( whichspace(addr) == shared_space ) {
+ addr= generic_to_shared(smid,addr);
+ space = shared_space;
+ }
+ decode_space(space,thread,src1,mem,addr);
+
+ type_info_key::type_decode(type,size,t);
+ if(debug_tensorcore)
+ printf("mma_st: thrd=%d,addr=%x, fp(size=%d), stride=%d\n",thrd,addr_reg.u32,size,src2_data.u32);
+ addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8;
+ addr_t push_addr;
+
+ ptx_reg_t nw_v[8];
+ for(k=0;k<8;k++){
+ if(k%2==0)
+ nw_v[k].s64=(v[k/2].s64&0xffff);
+ else
+ nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16);
+ }
+
+ for(k=0;k<8;k++){
+ if(type==F32_TYPE){
+ //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI);
+ push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride);
+ mem->write(push_addr,size/8,&v[k].s64,thread,pI);
+ mem_txn_addr[num_mem_txn++]=push_addr;
+
+ if(debug_tensorcore){
+ printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64);
+ float temp;
+ int l;
+ printf("thread=%d:",thrd);
+ for(l=0;l<8;l++){
+ temp=v[l].f32;
+ printf("%.2f",temp);
+ }
+ printf("\n");
+ }
+ }
+ else if(type==F16_TYPE){
+ if(wmma_layout==ROW){
+ //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI);
+ push_addr=new_addr+k*2;
+ mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI);
+ if(k%2==0)
+ mem_txn_addr[num_mem_txn++]=push_addr;
+ }
+ else if(wmma_layout==COL){
+ //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI);
+ push_addr=new_addr+k*2*stride;
+ mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI);
+ mem_txn_addr[num_mem_txn++]=push_addr;
+ }
+
+ if(debug_tensorcore)
+ printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64);
+ }
+ }
+
+ delete [] v;
+ inst.space = space;
+ inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn);
+
+ if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details
+ inst.data_size = 2; // 2 byte transaction
+ else
+ inst.data_size = 4; // 4 byte transaction
+
+ assert( inst.memory_op == insn_memory_op );
+ //thread->m_last_effective_address = addr;
+ //thread->m_last_memory_space = space;
+ }
+}
+
+void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst )
+{
+ size_t size;
+ int t,i;
+ unsigned smid;
+ const operand_info &dst = pI->dst();
+ const operand_info &src1 = pI->src1();
+ const operand_info &src2 = pI->src2();
+
+ unsigned type = pI->get_type();
+ unsigned wmma_type = pI->get_wmma_type();
+ unsigned wmma_layout = pI->get_wmma_layout(0);
+ int tid;
+ int thrd,stride;
+ ptx_thread_info *thread;
+
+
+ if(core->get_gpu()->is_functional_sim())
+ tid= inst.warp_id_func()*core->get_warp_size();
+ else
+ tid= inst.warp_id()*core->get_warp_size();
+
+ _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store;
+
+ for (thrd=0; thrd < core->get_warp_size(); thrd++){
+ thread = core->get_thread_info()[tid+thrd];
+ ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1);
+ ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1);
+ stride=src2_data.u32;
+ memory_space_t space = pI->get_space();
+
+ memory_space *mem = NULL;
+ addr_t addr = src1_data.u32;
+ smid = thread->get_hw_sid();
+ if( whichspace(addr) == shared_space ) {
+ addr= generic_to_shared(smid,addr);
+ space = shared_space;
+ }
+
+ decode_space(space,thread,src1,mem,addr);
+ type_info_key::type_decode(type,size,t);
+
+ ptx_reg_t data[16];
+ if(debug_tensorcore)
+ printf("mma_ld: thrd=%d,addr=%x, fpsize=%d, stride=%d\n",thrd,src1_data.u32,size,src2_data.u32);
+
+ addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8;
+ addr_t fetch_addr;
+ new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD];
+ int num_mem_txn=0;
+
+ if(wmma_type==LOAD_A){
+ for(i=0;i<16;i++){
+ if(wmma_layout==ROW){
+ //mem->read(new_addr+2*i,size/8,&data[i].s64);
+ fetch_addr=new_addr+2*i;
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ }
+ else if(wmma_layout==COL){
+ //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64);
+ fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4);
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ }
+ else{
+ printf("mma_ld:wrong_layout_type\n");
+ abort();
+
+ }
+ if(i%2==0)
+ mem_txn_addr[num_mem_txn++]=fetch_addr;
+ }
+ }
+ else if(wmma_type==LOAD_B){
+ for(i=0;i<16;i++){
+ if(wmma_layout==COL){
+ //mem->read(new_addr+2*i,size/8,&data[i].s64);
+ fetch_addr=new_addr+2*i;
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ }
+ else if(wmma_layout==ROW){
+ //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64);
+ fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4);
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ }
+ else{
+ printf("mma_ld:wrong_layout_type\n");
+ abort();
+ }
+ if(i%2==0)
+ mem_txn_addr[num_mem_txn++]=fetch_addr;
+ }
+ }
+ else if(wmma_type==LOAD_C){
+ for(i=0;i<8;i++){
+ if(type==F16_TYPE){
+ if(wmma_layout==ROW){
+ //mem->read(new_addr+2*i,size/8,&data[i].s64);
+ fetch_addr=new_addr+2*i;
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ if(i%2==0)
+ mem_txn_addr[num_mem_txn++]=fetch_addr;
+ }
+ else if(wmma_layout==COL){
+ //mem->read(new_addr+2*stride*i,size/8,&data[i].s64);
+ fetch_addr=new_addr+2*stride*i;
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ mem_txn_addr[num_mem_txn++]=fetch_addr;
+ }
+ else{
+ printf("mma_ld:wrong_type\n");
+ abort();
+ }
+ }
+ else if(type==F32_TYPE){
+ //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64);
+ fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride);
+ mem->read(fetch_addr,size/8,&data[i].s64);
+ mem_txn_addr[num_mem_txn++]=fetch_addr;
+ }
+ else{
+ printf("wrong type");
+ abort();
+ }
+ }
+ }
+ else{
+ printf("wrong wmma type\n");;
+ abort();
+ }
+ //generate timing memory request
+ inst.space = space;
+ inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn);
+
+ if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail.
+ inst.data_size = 2; // 2 byte transaction
+ else
+ inst.data_size = 4; // 4 byte transaction
+ assert( inst.memory_op == insn_memory_op );
+
+ if(debug_tensorcore){
+ if(type==F16_TYPE){
+ printf("\nmma_ld:thread%d= ",thrd);
+ for(i=0;i<16;i++){
+ printf("%x ",data[i].u64);
+ }
+ printf("\n");
+
+ printf("\nmma_ld:thread%d= ",thrd);
+ float temp;
+ for(i=0;i<16;i++){
+ temp=data[i].f16;
+ printf("%.2f ",temp);
+ }
+ printf("\n");
+ }
+ else{
+ printf("\nmma_ld:thread%d= ",thrd);
+ for(i=0;i<8;i++){
+ printf("%.2f ",data[i].f32);
+ }
+ printf("\n");
+ printf("\nmma_ld:thread%d= ",thrd);
+ for(i=0;i<8;i++){
+ printf("%x ",data[i].u64);
+ }
+ printf("\n");
+ }
+ }
+
+ if((wmma_type==LOAD_C)&&(type==F32_TYPE)){
+ thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]);
+ }
+ else{
+ ptx_reg_t nw_data[8];
+ int num_reg;
+
+ if(wmma_type==LOAD_C)
+ num_reg=4;
+ else
+ num_reg=8;
+
+ for(i=0;i<num_reg;i++){
+ nw_data[i].s64= ((data[2*i].s64 & 0xffff)<<16)| ((data[2*i+1].s64 & 0xffff));
+ }
+
+ if(wmma_type==LOAD_C)
+ thread->set_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]);
+ else
+ thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]);
+ if(debug_tensorcore){
+ printf("mma_ld:data[0].s64=%x,data[1].s64=%x,new_data[0].s64=%x\n",data[0].u64,data[1].u64,nw_data[0].u64);
+ printf("mma_ld:data[2].s64=%x,data[3].s64=%x,new_data[1].s64=%x\n",data[2].u64,data[3].u64,nw_data[1].u64);
+ printf("mma_ld:data[4].s64=%x,data[5].s64=%x,new_data[2].s64=%x\n",data[4].u64,data[5].u64,nw_data[2].u64);
+ printf("mma_ld:data[6].s64=%x,data[7].s64=%x,new_data[3].s64=%x\n",data[6].u64,data[7].u64,nw_data[3].u64);
+ if(wmma_type!=LOAD_C){
+ printf("mma_ld:data[8].s64=%x,data[9].s64=%x,new_data[4].s64=%x\n",data[8].u64,data[9].u64,nw_data[4].s64);
+ printf("mma_ld:data[10].s64=%x,data[11].s64=%x,new_data[5].s64=%x\n",data[10].u64,data[11].u64,nw_data[5].u64);
+ printf("mma_ld:data[12].s64=%x,data[13].s64=%x,new_data[6].s64=%x\n",data[12].u64,data[13].u64,nw_data[6].u64);
+ printf("mma_ld:data[14].s64=%x,data[15].s64=%x,new_data[7].s64=%x\n",data[14].u64,data[15].u64,nw_data[3].u64);
+ }
+ }
+ }
+
+ //thread->m_last_effective_address = addr;
+ //thread->m_last_memory_space = space;
+ }
+}
+
void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread )
{
ptx_reg_t a, d;
@@ -2605,9 +3562,24 @@ void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry
if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred;
else assert(0);
break;
- case F16_TYPE:
- assert(0);
- break;
+ case F16_TYPE:{
+ // assert(0);
+ // break;
+ assert( use_carry == false);
+ int orig_rm = fegetround();
+ switch ( rounding_mode ) {
+ case RN_OPTION: break;
+ case RZ_OPTION: fesetround( FE_TOWARDZERO ); break;
+ default: assert(0); break;
+ }
+ d.f16 = a.f16 * b.f16 + c.f16;
+ if ( pI->saturation_mode() ) {
+ if ( d.f16 < 0 ) d.f16 = 0;
+ else if ( d.f16 > 1.0f ) d.f16 = 1.0f;
+ }
+ fesetround( orig_rm );
+ break;
+ }
case F32_TYPE: {
assert( use_carry == false);
int orig_rm = fegetround();
@@ -2729,6 +3701,7 @@ void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread )
const operand_info &dst = pI->dst();
const operand_info &src1 = pI->src1();
unsigned i_type = pI->get_type();
+ assert( src1.is_param_local() == 0 );
if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) {
// pack or unpack operation
@@ -2907,9 +3880,25 @@ void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread )
if ( pI->is_lo() ) d.u64 = t.u64;
else assert(0);
break;
- case F16_TYPE:
- assert(0);
- break;
+ case F16_TYPE:{
+ //assert(0);
+ //break;
+ int orig_rm = fegetround();
+ switch ( rounding_mode ) {
+ case RN_OPTION: break;
+ case RZ_OPTION: fesetround( FE_TOWARDZERO ); break;
+ default: assert(0); break;
+ }
+
+ d.f16 = a.f16 * b.f16;
+
+ if ( pI->saturation_mode() ) {
+ if ( d.f16 < 0 ) d.f16 = 0;
+ else if ( d.f16 > 1.0f ) d.f16 = 1.0f;
+ }
+ fesetround( orig_rm );
+ break;
+ }
case F32_TYPE: {
int orig_rm = fegetround();
switch ( rounding_mode ) {
@@ -2972,7 +3961,7 @@ void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread )
case U32_TYPE:
case U64_TYPE:
assert(0); break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break;
case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break;
case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break;
default: assert(0); break;
@@ -3122,7 +4111,94 @@ void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread )
}
void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
-void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); }
+
+int prmt_mode_present(int mode)
+{
+ int returnval=0;
+ switch(mode){
+ case PRMT_F4E_MODE:
+ case PRMT_B4E_MODE:
+ case PRMT_RC8_MODE:
+ case PRMT_RC16_MODE:
+ case PRMT_ECL_MODE:
+ case PRMT_ECR_MODE:
+ returnval=1;
+ break;
+ default:
+ break;
+ }
+ return returnval;
+}
+int read_byte(int mode,int control,int d_sel_index,signed long long value){
+
+ int returnval;
+ int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}};
+ int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}};
+ int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}};
+ int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}};
+ int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}};
+ int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}};
+
+ if(!prmt_mode_present(mode)){
+ if(control&0x8){
+ returnval=0xff;
+ }
+ else{
+ returnval= (value>>(8*control)) & 0xff;
+ }
+ }
+ else{
+ switch(mode){
+ case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break;
+ case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break;
+ case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break;
+ case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break;
+ case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break;
+ case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break;
+ default: printf("ERROR\n");break;
+ }
+ }
+ return (returnval<<8*d_sel_index);
+}
+
+void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) {
+
+ ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data;
+ const operand_info &dst = pI->dst();
+ const operand_info &src1 = pI->src1();
+ const operand_info &src2 = pI->src2();
+ const operand_info &src3 = pI->src3();
+
+ unsigned mode = pI->prmt_op();
+ unsigned i_type = pI->get_type();
+
+ src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1);
+ src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1);
+ src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1);
+
+ tmpdata.s64=src1_data.s32|(src2_data.s64<<32);
+ int ctl[4];
+
+ if(!prmt_mode_present(mode)){
+ ctl[0]=(src3_data.s32>>0)&0xf;
+ ctl[1]=(src3_data.s32>>4)&0xf;
+ ctl[2]=(src3_data.s32>>8)&0xf;
+ ctl[3]=(src3_data.s32>>12)&0xf;
+ }
+ else{
+ ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3;
+ }
+
+ data.s32=0;
+ data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0
+ data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1
+ data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2
+ data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3
+
+ thread->set_operand_value(dst,data, i_type, thread, pI);
+
+
+}
void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread )
{
@@ -3165,7 +4241,21 @@ void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread )
src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1);
src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1);
- data.u64 = src1_data.u64 % src2_data.u64;
+ switch ( i_type ) {
+ case S32_TYPE:
+ data.s32 = src1_data.s32 % src2_data.s32;
+ break;
+ case S64_TYPE:
+ data.s64 = src1_data.s64 % src2_data.s64;
+ break;
+ case U32_TYPE:
+ data.u32 = src1_data.u32 % src2_data.u32;
+ break;
+ case U64_TYPE:
+ data.u64 = src1_data.u64 % src2_data.u64;
+ break;
+ default: assert(0); break;
+ }
thread->set_operand_value(dst,data, i_type, thread, pI);
}
@@ -3543,7 +4633,13 @@ void set_impl( const ptx_instruction *pI, ptx_thread_info *thread )
void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
{
unsigned i_type = pI->get_type();
- int tid = inst.warp_id() * core->get_warp_size();
+ int tid;
+
+ if(core->get_gpu()->is_functional_sim())
+ tid = inst.warp_id_func() * core->get_warp_size();
+ else
+ tid = inst.warp_id() * core->get_warp_size();
+
ptx_thread_info *thread = core->get_thread_info()[tid];
ptx_warp_info *warp_info = thread->m_warp_info;
int lane = warp_info->get_done_threads();
@@ -3833,6 +4929,88 @@ void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread )
thread->set_operand_value(dst,d, i_type, thread, pI);
}
+void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread )
+{
+ ptx_instruction * cpI = const_cast<ptx_instruction *>(pI); // constant
+ const operand_info &dst = cpI->dst();
+ const operand_info &src1 = pI->src1();
+ const operand_info &src2 = pI->src2();
+ const operand_info &src3 = pI->src3();
+ unsigned type = pI->get_type();
+ ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1);
+ ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1);
+ ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1);
+ ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1);
+ memory_space_t space = pI->get_space();
+ memory_space *mem = NULL;
+ addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0
+ ptx_cta_info *cta_info = thread->m_cta_info;
+
+ decode_space(space,thread,src1,mem,addr);
+
+ size_t size;
+ int t;
+ type_info_key::type_decode(type,size,t);
+
+ // store data in sstarr memory
+ mem->write(addr,size/8,&src3_data.s64,thread,pI);
+
+ // sync threads
+ cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15
+
+ thread->m_last_effective_address = addr;
+ thread->m_last_memory_space = space;
+ thread->m_last_dram_callback.function = bar_callback;
+ thread->m_last_dram_callback.instruction = cpI;
+
+ // the last thread that executes loads all of the data back from sstarr memory
+ int NUM_THREADS = cta_info->num_threads();
+ cta_info->inc_bar_threads();
+ if (NUM_THREADS == cta_info->get_bar_threads()) {
+ unsigned offset = 0;
+ addr = 0;
+ ptx_reg_t data;
+ float sstarr_fdata[NUM_THREADS];
+ signed long long sstarr_ldata[NUM_THREADS];
+ // loop through all of the threads
+ for (int tid = 0; tid < NUM_THREADS; tid++) {
+ data.u64=0;
+ mem->read(addr+(tid*4),size/8,&data.s64);
+ sstarr_fdata[tid] = data.f32;
+ sstarr_ldata[tid] = data.s64;
+ }
+
+ // squeeze the zeros out of the array and store data back into original array
+ mem = NULL;
+ addr = src1_data.u32;
+ space.set_type(global_space);
+ decode_space(space,thread,src1,mem,addr);
+ // store nonzero entries and indices
+ for (int tid = 0; tid < NUM_THREADS; tid++) {
+ if (sstarr_fdata[tid] != 0) {
+ float ftid = (float)tid;
+ mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI);
+ mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI);
+ offset++;
+ }
+ }
+ // store the number of nonzero elements in the array
+ data = thread->get_operand_value(src1, dst, type, thread, 1);
+ data.s64 += 4*(offset-1);
+ thread->set_operand_value(dst, data, type, thread, pI);
+
+ // fill the rest of the array with zeros (dst should always have a 0 in it)
+ while (offset < NUM_THREADS) {
+ mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI);
+ offset++;
+ }
+
+ cta_info->reset_bar_threads();
+ thread->m_last_effective_address = addr+(NUM_THREADS-1)*4;
+ thread->m_last_memory_space = space;
+ }
+}
+
void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread )
{
//printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n");
@@ -3861,7 +5039,7 @@ void st_impl( const ptx_instruction *pI, ptx_thread_info *thread )
if (!vector_spec) {
data = thread->get_operand_value(src1, dst, type, thread, 1);
mem->write(addr,size/8,&data.s64,thread,pI);
- } else {
+ } else {
if (vector_spec == V2_TYPE) {
ptx_reg_t* ptx_regs = new ptx_reg_t[2];
thread->get_vector_operand_values(src1, ptx_regs, 2);
@@ -3943,7 +5121,7 @@ void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread )
case B64_TYPE:
case U64_TYPE:
data.u64 = src1_data.u64 - src2_data.u64; break;
- case F16_TYPE: assert(0); break;
+ case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break;
case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break;
case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break;
default: assert(0); break;
@@ -4080,12 +5258,19 @@ void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread )
if (!ptx_tex_regs) ptx_tex_regs = new ptx_reg_t[4];
unsigned nelem = src2.get_vect_nelem();
thread->get_vector_operand_values(src2, ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture
-
+ /*
+ For programs with many streams, textures can be bound and unbound
+ asynchronously. This means we need to use the kernel's "snapshot" of
+ the state of the texture mappings when it was launched (so that we
+ don't try to access the incorrect texture mapping if it's been updated,
+ or that we don't access a mapping that has been unbound).
+ */
gpgpu_t *gpu = thread->get_gpu();
+ kernel_info_t &k = thread->get_kernel();
const struct textureReference* texref = gpu->get_texref(texname);
- const struct cudaArray* cuArray = gpu->get_texarray(texref);
- const struct textureInfo* texInfo = gpu->get_texinfo(texref);
- const struct textureReferenceAttr* texAttr = gpu->get_texattr(texref);
+ const struct cudaArray* cuArray = k.get_texarray(texname);
+ const struct textureInfo* texInfo = k.get_texinfo(texname);
+ const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname);
//assume always 2D f32 input
//access array with src2 coordinates
diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc
index 7bdf4d9..9554f55 100644
--- a/src/cuda-sim/memory.cc
+++ b/src/cuda-sim/memory.cc
@@ -44,9 +44,16 @@ template<unsigned BSIZE> memory_space_impl<BSIZE>::memory_space_impl( std::strin
assert( m_log2_block_size != (unsigned)-1 );
}
+template<unsigned BSIZE> void memory_space_impl<BSIZE>::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data)
+{
+ m_data[index].write(offset,length,(const unsigned char*)data);
+}
+
template<unsigned BSIZE> void memory_space_impl<BSIZE>::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI)
{
+
mem_addr_t index = addr >> m_log2_block_size;
+
if ( (addr+length) <= (index+1)*BSIZE ) {
// fast route for intra-block access
unsigned offset = addr & (BSIZE-1);
@@ -142,8 +149,9 @@ template<unsigned BSIZE> void memory_space_impl<BSIZE>::read( mem_addr_t addr, s
template<unsigned BSIZE> void memory_space_impl<BSIZE>::print( const char *format, FILE *fout ) const
{
typename map_t::const_iterator i_page;
- for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) {
- fprintf(fout, "%s - %#x:", m_name.c_str(), i_page->first);
+
+ for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) {
+ fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first);
i_page->second.print(format, fout);
}
}
diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h
index f785b8b..ab588bc 100644
--- a/src/cuda-sim/memory.h
+++ b/src/cuda-sim/memory.h
@@ -81,7 +81,7 @@ public:
{
unsigned int *i_data = (unsigned int*)m_data;
for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) {
- if (d % 8 == 0) {
+ if (d % 1 == 0) {
fprintf(fout, "\n");
}
fprintf(fout, format, i_data[d]);
@@ -104,6 +104,7 @@ class memory_space
public:
virtual ~memory_space() {}
virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0;
+ virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0;
virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0;
virtual void print( const char *format, FILE *fout ) const = 0;
virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0;
@@ -114,8 +115,10 @@ public:
memory_space_impl( std::string name, unsigned hash_size );
virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI );
+ virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data);
virtual void read( mem_addr_t addr, size_t length, void *data ) const;
virtual void print( const char *format, FILE *fout ) const;
+
virtual void set_watch( addr_t addr, unsigned watchpoint );
private:
diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def
index e1b1422..c4d6a83 100644
--- a/src/cuda-sim/opcodes.def
+++ b/src/cuda-sim/opcodes.def
@@ -52,6 +52,9 @@ OP_DEF(BRA_OP,bra_impl,"bra",0,3)
OP_DEF(BRX_OP,brx_impl,"brx",0,3)
OP_DEF(BREV_OP,brev_impl,"brev",1,1)
OP_DEF(BRKPT_OP,brkpt_impl,"brkpt",1,9)
+OP_W_DEF(MMA_OP,mma_impl,"mma",1,1)
+OP_W_DEF(MMA_LD_OP,mma_ld_impl,"mma_load",1,5)
+OP_W_DEF(MMA_ST_OP,mma_st_impl,"mma_store",0,5)
OP_DEF(CALL_OP,call_impl,"call",1,3)
OP_DEF(CALLP_OP,callp_impl,"callp",1,3)
OP_DEF(CLZ_OP,clz_impl,"clz",1,1)
@@ -60,6 +63,7 @@ OP_DEF(COS_OP,cos_impl,"cos",1,4)
OP_DEF(CVT_OP,cvt_impl,"cvt",1,1)
OP_DEF(CVTA_OP,cvta_impl,"cvta",1,1)
OP_DEF(DIV_OP,div_impl,"div",1,1)
+OP_DEF(DP4A_OP,dp4a_impl,"dp4a",1,1)
OP_DEF(EX2_OP,ex2_impl,"ex2",1,4)
OP_DEF(EXIT_OP,exit_impl,"exit",1,3)
OP_DEF(FMA_OP,fma_impl,"fma",1,2)
@@ -104,6 +108,7 @@ OP_DEF(SHR_OP,shr_impl,"shr",1,1)
OP_DEF(SIN_OP,sin_impl,"sin",1,4)
OP_DEF(SLCT_OP,slct_impl,"slct",1,1)
OP_DEF(SQRT_OP,sqrt_impl,"sqrt",1,4)
+OP_DEF(SST_OP,sst_impl,"sst",1,5)
OP_DEF(SSY_OP,ssy_impl,"ssy",0,3)
OP_DEF(ST_OP,st_impl,"st",0,5)
OP_DEF(SUB_OP,sub_impl,"sub",1,1)
diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h
index aa133da..86d3b99 100644
--- a/src/cuda-sim/opcodes.h
+++ b/src/cuda-sim/opcodes.h
@@ -60,5 +60,18 @@ enum special_regs {
WARPID_REG,
WARPSZ_REG
};
+enum wmma_type{
+ LOAD_A,
+ LOAD_B,
+ LOAD_C,
+ STORE_D,
+ MMA,
+ ROW,
+ COL,
+ M16N16K16,
+ M32N8K16,
+ M8N32K16
+
+};
#endif
diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l
index 5471d6f..36a8a4d 100644
--- a/src/cuda-sim/ptx.l
+++ b/src/cuda-sim/ptx.l
@@ -36,7 +36,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "ptx.tab.h"
#include <string.h>
-char linebuf[1024];
+#define LINEBUF_SIZE (64*1024)
+char linebuf[LINEBUF_SIZE];
unsigned col = 0;
#define TC col+=strlen(ptx_text);
#define CHECK_UNSIGNED \
@@ -60,7 +61,8 @@ addc TC; ptx_lval.int_value = ADDC_OP; return OPCODE;
and TC; ptx_lval.int_value = AND_OP; return OPCODE;
andn TC; ptx_lval.int_value = ANDN_OP; return OPCODE;
atom TC; ptx_lval.int_value = ATOM_OP; return OPCODE;
-bar TC; ptx_lval.int_value = BAR_OP; return OPCODE;
+bar.warp TC; ptx_lval.int_value = NOP_OP; return OPCODE;
+bar TC; ptx_lval.int_value = BAR_OP; return OPCODE;
bfe TC; ptx_lval.int_value = BFE_OP; return OPCODE;
bfi TC; ptx_lval.int_value = BFI_OP; return OPCODE;
bfind TC; ptx_lval.int_value = BFIND_OP; return OPCODE;
@@ -68,14 +70,20 @@ bra TC; ptx_lval.int_value = BRA_OP; return OPCODE;
brx TC; ptx_lval.int_value = BRX_OP; return OPCODE;
brev TC; ptx_lval.int_value = BREV_OP; return OPCODE;
brkpt TC; ptx_lval.int_value = BRKPT_OP; return OPCODE;
+
+wmma TC; ptx_lval.int_value = MMA_OP; return OPCODE;
+wmma\.load TC; ptx_lval.int_value = MMA_LD_OP; return OPCODE;
+wmma\.store TC; ptx_lval.int_value = MMA_ST_OP; return OPCODE;
+
call TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALL_OP; return OPCODE; // blocking opcode token in case the callee has the same name as an opcode
-callp TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALLP_OP; return OPCODE;
+callp TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALLP_OP; return OPCODE;
clz TC; ptx_lval.int_value = CLZ_OP; return OPCODE;
cnot TC; ptx_lval.int_value = CNOT_OP; return OPCODE;
cos TC; ptx_lval.int_value = COS_OP; return OPCODE;
cvt TC; ptx_lval.int_value = CVT_OP; return OPCODE;
cvta TC; ptx_lval.int_value = CVTA_OP; return OPCODE;
div TC; ptx_lval.int_value = DIV_OP; return OPCODE;
+dp4a TC; ptx_lval.int_value = DP4A_OP; return OPCODE;
ex2 TC; ptx_lval.int_value = EX2_OP; return OPCODE;
exit TC; ptx_lval.int_value = EXIT_OP; return OPCODE;
fma TC; ptx_lval.int_value = FMA_OP; return OPCODE;
@@ -121,6 +129,7 @@ shr TC; ptx_lval.int_value = SHR_OP; return OPCODE;
sin TC; ptx_lval.int_value = SIN_OP; return OPCODE;
slct TC; ptx_lval.int_value = SLCT_OP; return OPCODE;
sqrt TC; ptx_lval.int_value = SQRT_OP; return OPCODE;
+sst TC; ptx_lval.int_value = SST_OP; return OPCODE;
ssy TC; ptx_lval.int_value = SSY_OP; return OPCODE;
st TC; ptx_lval.int_value = ST_OP; return OPCODE;
st.volatile TC; ptx_lval.int_value = ST_OP; return OPCODE;
@@ -148,7 +157,26 @@ nop TC; ptx_lval.int_value = NOP_OP; return OPCODE;
break TC; ptx_lval.int_value = BREAK_OP; return OPCODE;
breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
+"CPTX_END" printf("ENDING CUSTOM PTX.\n"); BEGIN(IN_COMMENT);
+
<INITIAL,NOT_OPCODE,IN_INST,IN_FUNC_DECL>{
+\.a\.sync TC; ptx_lval.int_value = LOAD_A; return WMMA_DIRECTIVE;
+\.b\.sync TC; ptx_lval.int_value = LOAD_B; return WMMA_DIRECTIVE;
+\.c\.sync TC; ptx_lval.int_value = LOAD_C; return WMMA_DIRECTIVE;
+\.d\.sync TC; ptx_lval.int_value = STORE_D; return WMMA_DIRECTIVE;
+\.mma\.sync TC;ptx_lval.int_value=MMA; return WMMA_DIRECTIVE;
+
+\.row TC; ptx_lval.int_value = ROW; return LAYOUT;
+\.col TC; ptx_lval.int_value = COL; return LAYOUT;
+\.m16n16k16 TC; ptx_lval.int_value = M16N16K16; return CONFIGURATION;
+\.m32n8k16 TC; ptx_lval.int_value = M32N8K16; return CONFIGURATION;
+\.m8n32k16 TC; ptx_lval.int_value = M8N32K16; return CONFIGURATION;
+\.f4e TC; return PRMT_F4E_MODE;
+\.b4e TC; return PRMT_B4E_MODE;
+\.rc8 TC; return PRMT_RC8_MODE;
+\.ecl TC; return PRMT_ECL_MODE;
+\.ecr TC; return PRMT_ECR_MODE;
+\.rc16 TC; return PRMT_RC16_MODE;
\.align TC; return ALIGN_DIRECTIVE;
\.branchtargets TC; return BRANCHTARGETS_DIRECTIVE;
@@ -162,6 +190,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
\.file TC; BEGIN(INITIAL); return FILE_DIRECTIVE;
\.func TC; BEGIN(IN_FUNC_DECL); return FUNC_DIRECTIVE; // blocking opcode parsing in case the function has the same name as an opcode (e.g. sin(), cos())
\.global TC; return GLOBAL_DIRECTIVE;
+\.global.volatile TC; return GLOBAL_DIRECTIVE; //TODO: fix this!
\.local TC; return LOCAL_DIRECTIVE;
\.loc TC; return LOC_DIRECTIVE;
\.maxnctapersm TC; return MAXNCTAPERSM_DIRECTIVE;
@@ -175,6 +204,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
\.section TC; return SECTION_DIRECTIVE;
\.shared TC; return SHARED_DIRECTIVE;
\.sreg TC; return SREG_DIRECTIVE;
+\.sstarr TC; return SSTARR_DIRECTIVE;
\.struct TC; return STRUCT_DIRECTIVE;
\.surf TC; return SURF_DIRECTIVE; /* not in PTX 2.1 */
\.target TC; return TARGET_DIRECTIVE;
@@ -233,6 +263,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
\.u32 TC; return U32_TYPE;
\.u64 TC; return U64_TYPE;
\.f16 TC; return F16_TYPE;
+\.f16x2 TC; return F16_TYPE; /* TODO: figure out what this should really be */
\.f32 TC; return F32_TYPE;
\.f64 TC; return F64_TYPE;
\.ff64 TC; return FF64_TYPE;
@@ -382,7 +413,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
"//"[^\n]* TC; // eat single
-\n.* col=0; strncpy(linebuf, yytext + 1, 1024); yyless( 1 );
+\n.* col=0; strncpy(linebuf, yytext + 1, LINEBUF_SIZE); yyless( 1 );
" " TC;
"\t" TC;
@@ -399,9 +430,9 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
<IN_COMMENT>{
"*/" BEGIN(INITIAL);
"CPTX_BEGIN" printf("BEGINNING CUSTOM PTX.\n"); BEGIN(INITIAL);
-[^C*\n]+ // eat comment in chunks
-"C"
-"*" // eat the lone star
+[^C*\n]+ // eat comment in chunks
+"C" // eat the lone C
+"*" // eat the lone star
\n TC;
}
diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y
index c0c58a6..45392fb 100644
--- a/src/cuda-sim/ptx.y
+++ b/src/cuda-sim/ptx.y
@@ -37,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%token <string_value> STRING
%token <int_value> OPCODE
+%token <int_value> WMMA_DIRECTIVE
+%token <int_value> LAYOUT
+%token <int_value> CONFIGURATION
%token ALIGN_DIRECTIVE
%token BRANCHTARGETS_DIRECTIVE
%token BYTE_DIRECTIVE
@@ -47,7 +50,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%token PTR_DIRECTIVE
%token ENTRY_DIRECTIVE
%token EXTERN_DIRECTIVE
-%token WEAK_DIRECTIVE
%token FILE_DIRECTIVE
%token FUNC_DIRECTIVE
%token GLOBAL_DIRECTIVE
@@ -64,6 +66,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%token SECTION_DIRECTIVE
%token SHARED_DIRECTIVE
%token SREG_DIRECTIVE
+%token SSTARR_DIRECTIVE
%token STRUCT_DIRECTIVE
%token SURF_DIRECTIVE
%token TARGET_DIRECTIVE
@@ -199,6 +202,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
%token DOWN_OPTION;
%token BFLY_OPTION;
%token IDX_OPTION;
+%token PRMT_F4E_MODE;
+%token PRMT_B4E_MODE;
+%token PRMT_RC8_MODE;
+%token PRMT_RC16_MODE;
+%token PRMT_ECL_MODE;
+%token PRMT_ECR_MODE;
%type <int_value> function_decl_header
%type <ptr_value> function_decl
@@ -271,6 +280,7 @@ ptr_spec: /*empty*/
ptr_space_spec: GLOBAL_DIRECTIVE { add_ptr_spec(global_space); }
| LOCAL_DIRECTIVE { add_ptr_spec(local_space); }
| SHARED_DIRECTIVE { add_ptr_spec(shared_space); }
+ | CONST_DIRECTIVE { add_ptr_spec(global_space); }
ptr_align_spec: ALIGN_DIRECTIVE INT_OPERAND
@@ -332,6 +342,7 @@ var_spec_list: var_spec
var_spec: space_spec
| type_spec
| align_spec
+ | VISIBLE_DIRECTIVE
| EXTERN_DIRECTIVE { add_extern_spec(); }
| WEAK_DIRECTIVE
;
@@ -348,6 +359,7 @@ addressable_spec: CONST_DIRECTIVE { add_space_spec(const_space,$1); }
| LOCAL_DIRECTIVE { add_space_spec(local_space,0); }
| PARAM_DIRECTIVE { add_space_spec(param_space_unclassified,0); }
| SHARED_DIRECTIVE { add_space_spec(shared_space,0); }
+ | SSTARR_DIRECTIVE { add_space_spec(sstarr_space,0); }
| SURF_DIRECTIVE { add_space_spec(surf_space,0); }
| TEX_DIRECTIVE { add_space_spec(tex_space,0); }
;
@@ -427,6 +439,8 @@ option: type_spec
| compare_spec
| addressable_spec
| rounding_mode
+ | wmma_spec
+ | prmt_spec
| SYNC_OPTION { add_option(SYNC_OPTION); }
| ARRIVE_OPTION { add_option(ARRIVE_OPTION); }
| RED_OPTION { add_option(RED_OPTION); }
@@ -482,6 +496,7 @@ atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); }
rounding_mode: floating_point_rounding_mode
| integer_rounding_mode;
+
floating_point_rounding_mode: RN_OPTION { add_option(RN_OPTION); }
| RZ_OPTION { add_option(RZ_OPTION); }
| RM_OPTION { add_option(RM_OPTION); }
@@ -514,6 +529,24 @@ compare_spec:EQ_OPTION { add_option(EQ_OPTION); }
| NAN_OPTION { add_option(NAN_OPTION); }
;
+prmt_spec: PRMT_F4E_MODE { add_option( PRMT_F4E_MODE); }
+ | PRMT_B4E_MODE { add_option( PRMT_B4E_MODE); }
+ | PRMT_RC8_MODE { add_option( PRMT_RC8_MODE); }
+ | PRMT_RC16_MODE{ add_option( PRMT_RC16_MODE);}
+ | PRMT_ECL_MODE { add_option( PRMT_ECL_MODE); }
+ | PRMT_ECR_MODE { add_option( PRMT_ECR_MODE); }
+ ;
+
+wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_space_spec(global_space,0);add_ptr_spec(global_space); add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);}
+ | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);add_wmma_option($4);}
+ ;
+
+vp_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_space_spec(global_space,0);add_ptr_spec(global_space);add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);}
+ | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);add_wmma_option($4);}
+ ;
+
+
+
operand_list: operand
| operand COMMA operand_list;
@@ -542,6 +575,7 @@ operand: IDENTIFIER { add_scalar_operand( $1 ); }
vector_operand: LEFT_BRACE IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_2vector_operand($2,$4); }
| LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_3vector_operand($2,$4,$6); }
| LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_4vector_operand($2,$4,$6,$8); }
+ | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_8vector_operand($2,$4,$6,$8,$10,$12,$14,$16); }
| LEFT_BRACE IDENTIFIER RIGHT_BRACE { add_1vector_operand($2); }
;
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index ee36957..e4e0c09 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -177,8 +177,8 @@ void symbol_table::add_function( function_info *func, const char *filename, unsi
//Jin: handle instruction group for cdp
symbol_table* symbol_table::start_inst_group() {
- char inst_group_name[1024];
- snprintf(inst_group_name, 1024, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id);
+ char inst_group_name[4096];
+ snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id);
//previous added
assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end());
@@ -258,6 +258,14 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio
return prior_decl;
}
+function_info *symbol_table::lookup_function( std::string name )
+{
+ std::string key = std::string(name);
+ std::map<std::string,function_info*>::iterator it = m_function_info_lookup.find(key);
+ assert ( it != m_function_info_lookup.end() );
+ return it->second;
+}
+
type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec )
{
if( space_spec == param_space_unclassified )
@@ -281,8 +289,10 @@ type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_di
{
type_info_key t = base_type->get_key();
t.set_array_dim(array_dim);
- type_info *pt;
- pt = m_types[t] = new type_info(this,t);
+ type_info *pt = new type_info(this,t);
+ //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types.
+ //TODO: find where m_types can be used in future and solve the seg fault.
+ //pt = m_types[t] = new type_info(this,t);
return pt;
}
@@ -576,6 +586,40 @@ bool function_info::connect_break_targets() //connecting break instructions with
return modified;
}
+void function_info::do_pdom()
+{
+ create_basic_blocks();
+ connect_basic_blocks();
+ bool modified = false;
+ do {
+ find_dominators();
+ find_idominators();
+ modified = connect_break_targets();
+ } while (modified == true);
+
+ if ( g_debug_execution>=50 ) {
+ print_basic_blocks();
+ print_basic_block_links();
+ print_basic_block_dot();
+ }
+ if ( g_debug_execution>=2 ) {
+ print_dominators();
+ }
+ find_postdominators();
+ find_ipostdominators();
+ if ( g_debug_execution>=50 ) {
+ print_postdominators();
+ print_ipostdominators();
+ }
+ printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() );
+ for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions
+ ptx_instruction *pI = m_instr_mem[ii];
+ pI->pre_decode();
+ }
+ printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() );
+ fflush(stdout);
+ m_assembled = true;
+}
void intersect( std::set<int> &A, const std::set<int> &B )
{
// return intersection of A and B in A
@@ -996,7 +1040,7 @@ static std::list<operand_info> check_operands( int opcode,
const std::list<operand_info> &operands )
{
static int g_warn_literal_operands_two_type_inst;
- if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) ) {
+ if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) {
// just make sure these do not have have const operands...
if( !g_warn_literal_operands_two_type_inst ) {
std::list<operand_info>::const_iterator o;
@@ -1044,6 +1088,7 @@ ptx_instruction::ptx_instruction( int opcode,
const std::list<operand_info> &operands,
const operand_info &return_var,
const std::list<int> &options,
+ const std::list<int> &wmma_options,
const std::list<int> &scalar_type,
memory_space_t space_spec,
const char *file,
@@ -1062,6 +1107,7 @@ ptx_instruction::ptx_instruction( int opcode,
m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() );
m_return_var = return_var;
m_options = options;
+ m_wmma_options = wmma_options;
m_wide = false;
m_hi = false;
m_lo = false;
@@ -1079,9 +1125,35 @@ ptx_instruction::ptx_instruction( int opcode,
m_atomic_spec = 0;
m_membar_level = 0;
m_inst_size = 8; // bytes
-
+ int rr=0;
std::list<int>::const_iterator i;
unsigned n=1;
+ for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) {
+ int last_ptx_inst_option = *i;
+ switch ( last_ptx_inst_option ) {
+ case SYNC_OPTION:
+ case LOAD_A:
+ case LOAD_B:
+ case LOAD_C:
+ case STORE_D:
+ case MMA:
+ m_wmma_type=last_ptx_inst_option;
+ break;
+ case ROW:
+ case COL:
+ m_wmma_layout[rr++]=last_ptx_inst_option;
+ break;
+ case M16N16K16:
+ case M32N8K16:
+ case M8N32K16:
+ break;
+ default:
+ assert(0);
+ break;
+ }
+ }
+ rr=0;
+ n=1;
for ( i=options.begin(); i!= options.end(); i++, n++ ) {
int last_ptx_inst_option = *i;
switch ( last_ptx_inst_option ) {
@@ -1208,16 +1280,25 @@ ptx_instruction::ptx_instruction( int opcode,
case HALF_OPTION:
m_inst_size = 4; // bytes
break;
- case EXTP_OPTION:
- break;
- case NC_OPTION:
- break;
- case UP_OPTION:
- case DOWN_OPTION:
- case BFLY_OPTION:
- case IDX_OPTION:
+ case EXTP_OPTION:
+ break;
+ case NC_OPTION:
+ m_cache_option = last_ptx_inst_option;
+ break;
+ case UP_OPTION:
+ case DOWN_OPTION:
+ case BFLY_OPTION:
+ case IDX_OPTION:
m_shfl_op = last_ptx_inst_option;
break;
+ case PRMT_F4E_MODE:
+ case PRMT_B4E_MODE:
+ case PRMT_RC8_MODE:
+ case PRMT_ECL_MODE:
+ case PRMT_ECR_MODE:
+ case PRMT_RC16_MODE:
+ m_prmt_op = last_ptx_inst_option;
+ break;
default:
assert(0);
break;
@@ -1306,6 +1387,7 @@ function_info::function_info(int entry_point )
m_kernel_info.smem = 0;
m_local_mem_framesize = 0;
m_args_aligned_size = -1;
+ pdom_done = false; //initialize it to false
}
unsigned function_info::print_insn( unsigned pc, FILE * fp ) const
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 36ef3d5..1af85de 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -91,6 +91,7 @@ public:
bool is_tex() const { return m_space_spec == tex_space;}
bool is_func_addr() const { return m_is_function?true:false; }
int scalar_type() const { return m_scalar_type_spec;}
+ int get_alignment_spec() const { return m_alignment_spec;}
unsigned type_decode( size_t &size, int &t ) const;
static unsigned type_decode( int type, size_t &size, int &t );
memory_space_t get_memory_space() const { return m_space_spec; }
@@ -105,7 +106,7 @@ private:
int m_is_function;
bool m_is_non_arch_reg;
- friend class type_info_key_compare;
+ friend struct type_info_key_compare;
};
class symbol_table;
@@ -163,6 +164,7 @@ public:
m_is_global = false;
m_is_local = false;
m_is_param_local = false;
+ m_is_param_kernel = false;
m_is_tex = false;
m_is_func_addr = false;
m_reg_num_valid = false;
@@ -176,6 +178,7 @@ public:
if ( type ) m_is_global = type->get_key().is_global();
if ( type ) m_is_local = type->get_key().is_local();
if ( type ) m_is_param_local = type->get_key().is_param_local();
+ if ( type ) m_is_param_kernel = type->get_key().is_param_kernel();
if ( type ) m_is_tex = type->get_key().is_tex();
if ( type ) m_is_func_addr = type->get_key().is_func_addr();
}
@@ -222,10 +225,12 @@ public:
bool is_label() const { return m_is_label;}
bool is_shared() const { return m_is_shared;}
+ bool is_sstarr() const { return m_is_sstarr;}
bool is_const() const { return m_is_const;}
bool is_global() const { return m_is_global;}
bool is_local() const { return m_is_local;}
bool is_param_local() const { return m_is_param_local; }
+ bool is_param_kernel() const { return m_is_param_kernel; }
bool is_tex() const { return m_is_tex;}
bool is_func_addr() const { return m_is_func_addr; }
bool is_reg() const
@@ -279,10 +284,12 @@ private:
bool m_address_valid;
bool m_is_label;
bool m_is_shared;
+ bool m_is_sstarr;
bool m_is_const;
bool m_is_global;
bool m_is_local;
bool m_is_param_local;
+ bool m_is_param_kernel;
bool m_is_tex;
bool m_is_func_addr;
unsigned m_reg_num;
@@ -308,16 +315,19 @@ public:
symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line );
void add_function( function_info *func, const char *filename, unsigned linenumber );
bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table );
+ function_info *lookup_function(std::string name);
type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec );
type_info *add_type( function_info *func );
type_info *get_array_type( type_info *base_type, unsigned array_dim );
void set_label_address( const symbol *label, unsigned addr );
unsigned next_reg_num() { return ++m_reg_allocator;}
addr_t get_shared_next() { return m_shared_next;}
+ addr_t get_sstarr_next() { return m_sstarr_next;}
addr_t get_global_next() { return m_global_next;}
addr_t get_local_next() { return m_local_next;}
addr_t get_tex_next() { return m_tex_next;}
void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;}
+ void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;}
void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;}
void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;}
void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;}
@@ -339,6 +349,7 @@ public:
private:
unsigned m_reg_allocator;
unsigned m_shared_next;
+ unsigned m_sstarr_next;
unsigned m_const_next;
unsigned m_global_next;
unsigned m_local_next;
@@ -399,6 +410,8 @@ public:
m_type = symbolic_t;
} else if ( addr->is_param_local() ) {
m_type = symbolic_t;
+ } else if ( addr->is_param_kernel() ) {
+ m_type = symbolic_t;
} else if ( addr->is_tex() ) {
m_type = symbolic_t;
} else if ( addr->is_func_addr() ) {
@@ -429,11 +442,15 @@ public:
m_uid = get_uid();
m_valid = true;
m_type = memory_t;
- m_value.m_vector_symbolic = new const symbol*[4];
+ m_value.m_vector_symbolic = new const symbol*[8];
m_value.m_vector_symbolic[0] = addr1;
m_value.m_vector_symbolic[1] = addr2;
m_value.m_vector_symbolic[2] = NULL;
m_value.m_vector_symbolic[3] = NULL;
+ m_value.m_vector_symbolic[4] = NULL;
+ m_value.m_vector_symbolic[5] = NULL;
+ m_value.m_vector_symbolic[6] = NULL;
+ m_value.m_vector_symbolic[7] = NULL;
m_addr_offset = 0;
m_vector = false;
m_neg_pred = false;
@@ -567,16 +584,48 @@ public:
m_valid = true;
m_vector = true;
m_type = vector_t;
- m_value.m_vector_symbolic = new const symbol*[4];
+ m_value.m_vector_symbolic = new const symbol*[8];
m_value.m_vector_symbolic[0] = s1;
m_value.m_vector_symbolic[1] = s2;
m_value.m_vector_symbolic[2] = s3;
m_value.m_vector_symbolic[3] = s4;
+ m_value.m_vector_symbolic[4] = NULL;
+ m_value.m_vector_symbolic[5] = NULL;
+ m_value.m_vector_symbolic[6] = NULL;
+ m_value.m_vector_symbolic[7] = NULL;
m_addr_offset = 0;
m_neg_pred = false;
m_is_return_var = false;
m_immediate_address=false;
}
+ operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8)
+ {
+ init();
+ m_is_non_arch_reg = false;
+ m_addr_space = undefined_space;
+ m_operand_lohi = 0;
+ m_double_operand_type = 0;
+ m_operand_neg = false;
+ m_const_mem_offset = 0;
+ m_uid = get_uid();
+ m_valid = true;
+ m_vector = true;
+ m_type = vector_t;
+ m_value.m_vector_symbolic = new const symbol*[8];
+ m_value.m_vector_symbolic[0] = s1;
+ m_value.m_vector_symbolic[1] = s2;
+ m_value.m_vector_symbolic[2] = s3;
+ m_value.m_vector_symbolic[3] = s4;
+ m_value.m_vector_symbolic[4] = s5;
+ m_value.m_vector_symbolic[5] = s6;
+ m_value.m_vector_symbolic[6] = s7;
+ m_value.m_vector_symbolic[7] = s8;
+ m_addr_offset = 0;
+ m_neg_pred = false;
+ m_is_return_var = false;
+ m_immediate_address=false;
+ }
+
void init()
{
m_uid=(unsigned)-1;
@@ -623,12 +672,16 @@ public:
if( !m_value.m_vector_symbolic[1] ) return 1;
if( !m_value.m_vector_symbolic[2] ) return 2;
if( !m_value.m_vector_symbolic[3] ) return 3;
- return 4;
+ if( !m_value.m_vector_symbolic[4] ) return 4;
+ if( !m_value.m_vector_symbolic[5] ) return 5;
+ if( !m_value.m_vector_symbolic[6] ) return 6;
+ if( !m_value.m_vector_symbolic[7] ) return 7;
+ return 8;
}
const symbol* vec_symbol(int idx) const
{
- assert(idx < 4);
+ assert(idx < 8);
const symbol *result = m_value.m_vector_symbolic[idx];
assert( result != NULL );
return result;
@@ -675,6 +728,13 @@ public:
return m_value.m_symbolic->type()->get_key().is_param_local();
}
+ bool is_param_kernel() const
+ {
+ if ( m_type != symbolic_t )
+ return false;
+ return m_value.m_symbolic->type()->get_key().is_param_kernel();
+ }
+
bool is_vector() const
{
if ( m_vector) return true;
@@ -685,6 +745,10 @@ public:
int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();}
int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; }
int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; }
+ int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; }
+ int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; }
+ int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; }
+ int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; }
int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); }
int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; }
bool is_label() const { return m_type == label_t;}
@@ -713,6 +777,7 @@ public:
}
return m_value.m_symbolic->is_shared();
}
+ bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();}
bool is_const() const { return m_value.m_symbolic->is_const();}
bool is_global() const { return m_value.m_symbolic->is_global();}
bool is_local() const { return m_value.m_symbolic->is_local();}
@@ -731,7 +796,7 @@ public:
{
ptx_reg_t result;
switch ( m_type ) {
- case int_t: result.s32 = m_value.m_int; break;
+ case int_t: result.s64 = m_value.m_int; break;
case float_op_t: result.f32 = m_value.m_float; break;
case double_op_t: result.f64 = m_value.m_double; break;
case unsigned_t: result.u32 = m_value.m_unsigned; break;
@@ -860,6 +925,7 @@ public:
const std::list<operand_info> &operands,
const operand_info &return_var,
const std::list<int> &options,
+ const std::list<int> &wmma_options,
const std::list<int> &scalar_type,
memory_space_t space_spec,
const char *file,
@@ -942,6 +1008,31 @@ public:
assert( m_operands.size() > 3 );
return m_operands[3];
}
+ const operand_info &src4() const
+ {
+ assert( m_operands.size() > 4 );
+ return m_operands[4];
+ }
+ const operand_info &src5() const
+ {
+ assert( m_operands.size() > 5 );
+ return m_operands[5];
+ }
+ const operand_info &src6() const
+ {
+ assert( m_operands.size() > 6 );
+ return m_operands[6];
+ }
+ const operand_info &src7() const
+ {
+ assert( m_operands.size() > 7 );
+ return m_operands[7];
+ }
+ const operand_info &src8() const
+ {
+ assert( m_operands.size() > 8 );
+ return m_operands[8];
+ }
const operand_info &operand_lookup( unsigned n ) const
{
@@ -957,6 +1048,12 @@ public:
unsigned get_vector() const { return m_vector_spec;}
unsigned get_atomic() const { return m_atomic_spec;}
+ int get_wmma_type() const {
+ return m_wmma_type;
+ }
+ int get_wmma_layout(int index) const {
+ return m_wmma_layout[index];//0->Matrix D,1->Matrix C
+ }
int get_type() const
{
assert( !m_scalar_type.empty() );
@@ -1004,13 +1101,14 @@ public:
unsigned dimension() const { return m_geom_spec;}
unsigned barrier_op() const {return m_barrier_op;}
unsigned shfl_op() const {return m_shfl_op;}
+ unsigned prmt_op() const {return m_prmt_op;}
enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot };
enum vote_mode_t vote_mode() const { return m_vote_mode; }
int membar_level() const { return m_membar_level; }
bool has_memory_read() const {
- if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP )
+ if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP )
return true;
// Check PTXPlus operand type below
// Source operands are memory operands
@@ -1022,7 +1120,7 @@ public:
return false;
}
bool has_memory_write() const {
- if( m_opcode == ST_OP ) return true;
+ if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true;
// Check PTXPlus operand type below
// Destination operand is a memory operand
ptx_instruction::const_iterator op=op_iter_begin();
@@ -1056,6 +1154,7 @@ private:
operand_info m_return_var;
std::list<int> m_options;
+ std::list<int> m_wmma_options;
bool m_wide;
bool m_hi;
bool m_lo;
@@ -1065,11 +1164,15 @@ private:
bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps)
bool m_to_option;
unsigned m_cache_option;
+ int m_wmma_type;
+ int m_wmma_layout[2];
+ int m_wmma_configuration;
unsigned m_rounding_mode;
unsigned m_compare_op;
unsigned m_saturation_mode;
unsigned m_barrier_op;
unsigned m_shfl_op;
+ unsigned m_prmt_op;
std::list<int> m_scalar_type;
memory_space_t m_space_spec;
@@ -1178,7 +1281,7 @@ public:
//Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15
void find_ipostdominators( );
void print_ipostdominators();
-
+ void do_pdom(); //function to call pdom analysis
unsigned get_num_reconvergence_pairs();
@@ -1242,6 +1345,7 @@ public:
void finalize( memory_space *param_mem );
void param_to_shared( memory_space *shared_mem, symbol_table *symtab );
void list_param( FILE *fout ) const;
+ void ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ;
const struct gpgpu_ptx_sim_info* get_kernel_info () const
{
@@ -1277,6 +1381,20 @@ public:
m_local_mem_framesize = sz;
}
bool is_entry_point() const { return m_entry_point; }
+ bool is_pdom_set() const { return pdom_done; } //return pdom flag
+ void set_pdom() { pdom_done = true; } //set pdom flag
+
+ void add_config_param( size_t size, unsigned alignment ){
+ unsigned offset = 0;
+ if (m_param_configs.size()>0){
+ unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second;
+ //ensure offset matches alignment requirements
+ offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom;
+ }
+ m_param_configs.push_back(std::pair<size_t,unsigned>(size, offset));
+ }
+
+ std::pair<size_t, unsigned> get_param_config(unsigned param_num) const { return m_param_configs[param_num]; }
void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;}
unsigned get_maxnt_id() { return maxnt_id;}
@@ -1288,12 +1406,14 @@ private:
bool m_entry_point;
bool m_extern;
bool m_assembled;
+ bool pdom_done; //flag to check whether pdom is completed or not
std::string m_name;
ptx_instruction **m_instr_mem;
unsigned m_start_PC;
unsigned m_instr_mem_size;
std::map<std::string,param_t> m_kernel_params;
std::map<unsigned,param_info> m_ptx_kernel_param_info;
+ std::vector< std::pair<size_t, unsigned> > m_param_configs;
const symbol *m_return_var_sym;
std::vector<const symbol*> m_args;
std::list<ptx_instruction*> m_instructions;
@@ -1312,6 +1432,8 @@ private:
//parameter size for device kernels
int m_args_aligned_size;
+
+ addr_t m_n; // offset in m_instr_mem (used in do_pdom)
};
class arg_buffer_t {
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 6c1b595..8c6f361 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -32,6 +32,7 @@
#include <unistd.h>
#include <dirent.h>
#include <fstream>
+#include <sstream>
/// globals
@@ -54,6 +55,7 @@ extern int ptxinfo_debug;
extern FILE *ptxinfo_in;
static bool g_save_embedded_ptx;
+static int g_occupancy_sm_number;
bool g_keep_intermediate_files;
bool m_ptx_save_converted_ptxplus;
@@ -71,6 +73,10 @@ void ptx_reg_options(option_parser_t opp)
&m_ptx_save_converted_ptxplus,
"Saved converted ptxplus to a file",
"0");
+ option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number,
+ "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. "
+ "This parameter is required in the config.",
+ "0");
}
void print_ptx_file( const char *p, unsigned source_num, const char *filename )
@@ -120,7 +126,7 @@ char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilenam
fflush(stdout);
printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline);
result = system(commandline);
- if(result){printf("GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);}
+ if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);}
// Get ptxplus from file
@@ -142,7 +148,7 @@ char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilenam
printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline);
int rm_result = system(rm_commandline);
if( rm_result != 0 ) {
- printf("GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result);
+ fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result);
exit(1);
}
}
@@ -184,6 +190,13 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source
return symtab;
}
+symbol_table *gpgpu_ptx_sim_load_ptx_from_filename( const char *filename )
+{
+ symbol_table *symtab=init_parser(filename);
+ printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename);
+ return symtab;
+}
+
void fix_duplicate_errors(char fname2[1024]) {
char tempfile[1024] = "_temp_ptx";
char commandline[1024];
@@ -193,7 +206,7 @@ void fix_duplicate_errors(char fname2[1024]) {
printf("Running: %s\n", commandline);
int result = system(commandline);
if (result != 0) {
- printf("GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile);
+ fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile);
exit(1);
}
@@ -282,54 +295,115 @@ void fix_duplicate_errors(char fname2[1024]) {
printf("Running: %s\n", commandline);
result = system(commandline);
if (result != 0) {
- printf("GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile);
+ fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile);
exit(1);
}
}
-void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version )
+
+//we need the application name here too.
+char* get_app_binary_name(){
+ char exe_path[1025];
+ char *self_exe_path;
+#ifdef __APPLE__
+ //AMRUTH: get apple device and check the result.
+ printf("WARNING: not tested for Apple-mac devices \n");
+ abort();
+#else
+ std::stringstream exec_link;
+ exec_link << "/proc/self/exe";
+ ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024);
+ assert(path_length != -1);
+ exe_path[path_length] = '\0';
+
+ char *token = strtok(exe_path, "/");
+ while(token !=NULL){
+ self_exe_path = token;
+ token = strtok(NULL,"/");
+ }
+#endif
+ self_exe_path = strtok(self_exe_path, ".");
+ printf("self exe links to: %s\n", self_exe_path);
+ return self_exe_path;
+}
+
+void gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version)
{
- char fname[1024];
- snprintf(fname,1024,"_ptx_XXXXXX");
- int fd=mkstemp(fname);
- close(fd);
+ std::string ptxas_filename(std::string(filename) + "as");
+ char buff[1024], extra_flags[1024];
+ extra_flags[0]=0;
+ extern bool g_cdp_enabled;
+ if(!g_cdp_enabled)
+ snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+ else
+ snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
+ snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s",
+ extra_flags, filename, ptxas_filename.c_str());
+ int result = system(buff);
+ if( result != 0 ) {
+ printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result);
+ printf(" Ensure ptxas is in your path.\n");
+ exit(1);
+ }
- printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname);
- FILE *ptxfile = fopen(fname,"w");
- fprintf(ptxfile,"%s", p_for_info);
- fclose(ptxfile);
+ g_ptxinfo_filename = strdup(ptxas_filename.c_str());
+ ptxinfo_in = fopen(g_ptxinfo_filename,"r");
+ ptxinfo_parse();
+ fclose(ptxinfo_in);
+}
- char fname2[1024];
- snprintf(fname2,1024,"_ptx2_XXXXXX");
- fd=mkstemp(fname2);
- close(fd);
- char commandline2[4096];
- snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2);
- printf("Running: %s\n", commandline2);
- int result = system(commandline2);
- if( result != 0 ) {
- printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result);
- printf(" Ensure you have write access to simulation directory\n");
- printf(" and have \'cat\' and \'sed\' in your path.\n");
- exit(1);
- }
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version )
+{
+ //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations.
+ char ptx_file[1000];
+ char *name=get_app_binary_name();
+ char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024];
+ for (int index=1; index <= no_of_ptx; index++){
+ snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version);
+ snprintf(fname,1024,"_ptx_XXXXXX");
+ int fd=mkstemp(fname);
+ close(fd);
- char tempfile_ptxinfo[1024];
- snprintf(tempfile_ptxinfo,1024,"%sinfo",fname);
- char commandline[1024];
- char extra_flags[1024];
- extra_flags[0]=0;
+ printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname);
+ snprintf(commandline,4096,"cat %s > %s",ptx_file, fname);
+ if (system(commandline) !=0) {
+ printf("ERROR: %s command failed\n", commandline);
+ exit(0);
+ }
+
+ snprintf(fname2,1024,"_ptx2_XXXXXX");
+ fd=mkstemp(fname2);
+ close(fd);
+ char commandline2[4096];
+ snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2);
+ printf("Running: %s\n", commandline2);
+ int result = system(commandline2);
+ if( result != 0 ) {
+ printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result);
+ printf(" Ensure you have write access to simulation directory\n");
+ printf(" and have \'cat\' and \'sed\' in your path.\n");
+ exit(1);
+ }
+
+ snprintf(tempfile_ptxinfo,1024,"%sinfo",fname);
+ char extra_flags[1024];
+ extra_flags[0]=0;
#if CUDART_VERSION >= 3000
- if (sm_version == 0) sm_version = 20;
+ if ( g_occupancy_sm_number == 0 ) {
+ fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n"
+ "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n"
+ "A register size/SM mismatch may result in occupancy differences." );
+ exit(1);
+ }
extern bool g_cdp_enabled;
if(!g_cdp_enabled)
- snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+ snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number);
else
- snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
+ snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number);
#endif
- snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s",
+ snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s",
extra_flags, fname2, tempfile_ptxinfo);
printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
result = system(commandline);
@@ -345,26 +419,100 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
extra_flags, fname2, tempfile_ptxinfo);
printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline);
result = system(commandline);
+ }
+ if (result != 0) {
+ printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result);
+ printf(" Ensure ptxas is in your path.\n");
+ exit(1);
+ }
+ }
+ }
+
+ //TODO: duplicate code! move it into a function so that it can be reused!
+ if(no_of_ptx==0) {
+ //For CDP, we dump everything. So no_of_ptx will be 0.
+ snprintf(fname,1024,"_ptx_XXXXXX");
+ int fd=mkstemp(fname);
+ close(fd);
+
+ printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname);
+ FILE *ptxfile = fopen(fname,"w");
+ fprintf(ptxfile,"%s", p_for_info);
+ fclose(ptxfile);
+
+ snprintf(fname2,1024,"_ptx2_XXXXXX");
+ fd=mkstemp(fname2);
+ close(fd);
+ char commandline2[4096];
+ snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2);
+ printf("Running: %s\n", commandline2);
+ int result = system(commandline2);
+ if( result != 0 ) {
+ printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result);
+ printf(" Ensure you have write access to simulation directory\n");
+ printf(" and have \'cat\' and \'sed\' in your path.\n");
+ exit(1);
}
- if (result != 0) {
+ //char tempfile_ptxinfo[1024];
+ snprintf(tempfile_ptxinfo,1024,"%sinfo",fname);
+ char extra_flags[1024];
+ extra_flags[0]=0;
+
+ #if CUDART_VERSION >= 3000
+ if (sm_version == 0) sm_version = 20;
+ extern bool g_cdp_enabled;
+ if(!g_cdp_enabled)
+ snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+ else
+ snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
+ #endif
+
+ snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s",
+ extra_flags, fname2, tempfile_ptxinfo);
+ printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
+ fflush(stdout);
+ result = system(commandline);
+ if( result != 0 ) {
printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result);
printf(" Ensure ptxas is in your path.\n");
exit(1);
}
}
- ptxinfo_in = fopen(tempfile_ptxinfo,"r");
- g_ptxinfo_filename = tempfile_ptxinfo;
+ //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual.
+ if(no_of_ptx>0){
+ char commandline3[4096];
+ snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx");
+ snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo);
+ if (system(commandline3)!=0) {
+ printf("ERROR: Either we dont have info files or cat is not working \n");
+ printf("ERROR: %s command failed\n",commandline3);
+ exit(1);
+ }
+ }
+
+ if(no_of_ptx>0)
+ g_ptxinfo_filename = final_tempfile_ptxinfo;
+ else
+ g_ptxinfo_filename = tempfile_ptxinfo;
+ ptxinfo_in = fopen(g_ptxinfo_filename,"r");
+
ptxinfo_parse();
+ snprintf(commandline,1024,"rm -f *info");
+ if( system(commandline) != 0 ) {
+ printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n");
+ exit(1);
+ }
if( ! g_save_embedded_ptx ) {
- snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo);
+ if(no_of_ptx>0)
+ snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo);
+ else
+ snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo);
printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline);
- result = system(commandline);
- if( result != 0 ) {
- printf("GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result);
+ if( system(commandline) != 0 ) {
+ printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n");
exit(1);
}
}
-}
-
+} \ No newline at end of file
diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h
index d3d0c92..e5df6a9 100644
--- a/src/cuda-sim/ptx_loader.h
+++ b/src/cuda-sim/ptx_loader.h
@@ -30,9 +30,12 @@
#include <string>
extern bool g_override_embedded_ptx;
+extern int no_of_ptx; //counter to track number of ptx files to be extracted in an application.
class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num );
+class symbol_table *gpgpu_ptx_sim_load_ptx_from_filename( const char *filename );
void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version=20 );
+void gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version );
char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str);
bool keep_intermediate_files();
diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc
index e5731a8..25758dd 100644
--- a/src/cuda-sim/ptx_parser.cc
+++ b/src/cuda-sim/ptx_parser.cc
@@ -32,6 +32,8 @@
extern int ptx_error( const char *s );
extern int ptx_lineno;
+extern int ptx_parse();
+extern FILE *ptx_in;
static const struct core_config *g_shader_core_config;
void set_ptx_warp_size(const struct core_config * warp_size)
@@ -59,6 +61,7 @@ memory_space_t g_ptr_spec = undefined_space;
int g_scalar_type_spec = -1;
int g_vector_spec = -1;
int g_alignment_spec = -1;
+int g_size = -1;
int g_extern_spec = 0;
// variable declaration stuff:
@@ -72,6 +75,7 @@ symbol *g_label;
int g_opcode = -1;
std::list<operand_info> g_operands;
std::list<int> g_options;
+std::list<int> g_wmma_options;
std::list<int> g_scalar_type;
#define PTX_PARSE_DPRINTF(...) \
@@ -105,6 +109,36 @@ void read_parser_environment_variables()
}
}
+void init_directive_state()
+{
+ PTX_PARSE_DPRINTF("init_directive_state");
+ g_space_spec=undefined_space;
+ g_ptr_spec=undefined_space;
+ g_scalar_type_spec=-1;
+ g_vector_spec=-1;
+ g_opcode=-1;
+ g_alignment_spec = -1;
+ g_size = -1;
+ g_extern_spec = 0;
+ g_scalar_type.clear();
+ g_operands.clear();
+ g_last_symbol = NULL;
+}
+
+void init_instruction_state()
+{
+ PTX_PARSE_DPRINTF("init_instruction_state");
+ g_pred = NULL;
+ g_neg_pred = 0;
+ g_pred_mod = -1;
+ g_label = NULL;
+ g_opcode = -1;
+ g_options.clear();
+ g_wmma_options.clear();
+ g_return_var = operand_info();
+ init_directive_state();
+}
+
symbol_table *init_parser( const char *ptx_filename )
{
g_filename = strdup(ptx_filename);
@@ -112,9 +146,9 @@ symbol_table *init_parser( const char *ptx_filename )
g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL);
g_global_symbol_table = g_current_symbol_table = g_global_allfiles_symbol_table;
}
- else {
+ /*else {
g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table);
- }
+ }*/
ptx_lineno = 1;
#define DEF(X,Y) g_ptx_token_decode[X] = Y;
@@ -135,35 +169,13 @@ symbol_table *init_parser( const char *ptx_filename )
g_ptx_token_decode[generic_space] = "generic_space";
g_ptx_token_decode[instruction_space] = "instruction_space";
- return g_global_symbol_table;
-}
-
-void init_directive_state()
-{
- PTX_PARSE_DPRINTF("init_directive_state");
- g_space_spec=undefined_space;
- g_ptr_spec=undefined_space;
- g_scalar_type_spec=-1;
- g_vector_spec=-1;
- g_opcode=-1;
- g_alignment_spec = -1;
- g_extern_spec = 0;
- g_scalar_type.clear();
- g_operands.clear();
- g_last_symbol = NULL;
-}
-
-void init_instruction_state()
-{
- PTX_PARSE_DPRINTF("init_instruction_state");
- g_pred = NULL;
- g_neg_pred = 0;
- g_pred_mod = -1;
- g_label = NULL;
- g_opcode = -1;
- g_options.clear();
- g_return_var = operand_info();
init_directive_state();
+ init_instruction_state();
+
+ ptx_in = fopen(ptx_filename, "r");
+ ptx_parse();
+ fclose(ptx_in);
+ return g_global_symbol_table;
}
static int g_entry_point;
@@ -265,7 +277,7 @@ void parse_assert_impl( int test_value, const char *file, unsigned line, const c
parse_error_impl(file,line, msg);
}
-extern char linebuf[1024];
+extern char linebuf[4096];
void set_return()
@@ -300,6 +312,7 @@ void add_instruction()
g_operands,
g_return_var,
g_options,
+ g_wmma_options,
g_scalar_type,
g_space_spec,
g_filename,
@@ -367,6 +380,9 @@ int pad_address (new_addr_type address, unsigned size, unsigned maxalign) {
void add_identifier( const char *identifier, int array_dim, unsigned array_ident )
{
+ if(array_ident==ARRAY_IDENTIFIER){
+ g_size *= array_dim;
+ }
if( g_func_decl && (g_func_info == NULL) ) {
// return variable decl...
assert( g_add_identifier_cached__identifier == NULL );
@@ -433,13 +449,27 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
assert( (num_bits%8) == 0 );
addr = g_current_symbol_table->get_shared_next();
addr_pad = pad_address(addr, num_bits/8, 128);
- printf("from 0x%x to 0x%lx (shared memory space)\n",
+ printf("from 0x%llx to 0x%llx (shared memory space)\n",
addr+addr_pad,
addr+addr_pad + num_bits/8);
fflush(stdout);
g_last_symbol->set_address( addr+addr_pad );
g_current_symbol_table->alloc_shared( num_bits/8 + addr_pad );
break;
+ case sstarr_space:
+ printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ",
+ identifier);
+ fflush(stdout);
+ assert( (num_bits%8) == 0 );
+ addr = g_current_symbol_table->get_sstarr_next();
+ addr_pad = pad_address(addr, num_bits/8, 128);
+ printf("from 0x%x to 0x%lx (sstarr memory space)\n",
+ addr+addr_pad,
+ addr+addr_pad + num_bits/8);
+ fflush(stdout);
+ g_last_symbol->set_address( addr+addr_pad );
+ g_current_symbol_table->alloc_sstarr( num_bits/8 + addr_pad );
+ break;
case const_space:
if( array_ident == ARRAY_IDENTIFIER_NO_DIM ) {
printf("GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" (need size information)\n", identifier );
@@ -450,7 +480,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
assert( (num_bits%8) == 0 );
addr = g_current_symbol_table->get_global_next();
addr_pad = pad_address(addr, num_bits/8, 128);
- printf("from 0x%x to 0x%lx (global memory space) %u\n",
+ printf("from 0x%llx to 0x%llx (global memory space) %u\n",
addr+addr_pad,
addr+addr_pad + num_bits/8,
g_const_alloc++);
@@ -471,7 +501,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
assert( (num_bits%8) == 0 );
addr = g_current_symbol_table->get_global_next();
addr_pad = pad_address(addr, num_bits/8, 128);
- printf("from 0x%x to 0x%lx (global memory space)\n",
+ printf("from 0x%llx to 0x%llx (global memory space)\n",
addr+addr_pad,
addr+addr_pad + num_bits/8);
fflush(stdout);
@@ -488,7 +518,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
assert( (num_bits%8) == 0 );
addr = g_current_symbol_table->get_local_next();
addr_pad = pad_address(addr, num_bits/8, 128);
- printf("from 0x%x to 0x%lx (local memory space)\n",
+ printf("from 0x%llx to 0x%llx (local memory space)\n",
addr+addr_pad,
addr+addr_pad + num_bits/8);
fflush(stdout);
@@ -501,7 +531,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
assert( (num_bits%8) == 0 );
addr = g_current_symbol_table->get_local_next();
addr_pad = pad_address(addr, num_bits/8, 128);
- printf("from 0x%x to 0x%lx\n",
+ printf("from 0x%llx to 0x%llx\n",
addr+addr_pad,
addr+addr_pad + num_bits/8);
fflush(stdout);
@@ -556,10 +586,15 @@ void add_constptr(const char* identifier1, const char* identifier2, int offset)
void add_function_arg()
{
+ assert(g_size>0);
if( g_func_info ) {
PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() );
g_func_info->add_arg(g_last_symbol);
+ unsigned alignment = (g_alignment_spec==-1) ? g_size : g_alignment_spec;
+ assert(alignment==1||alignment==2||alignment==4||alignment==8||alignment==16);//known valid alignment values
+ g_func_info->add_config_param( g_size, alignment);
}
+
}
void add_extern_spec()
@@ -611,12 +646,38 @@ void add_vector_spec(int spec )
void add_scalar_type_spec( int type_spec )
{
+ //save size of parameter
+ switch ( type_spec ) {
+ case B8_TYPE:
+ case S8_TYPE:
+ case U8_TYPE:
+ g_size = 1; break;
+ case B16_TYPE:
+ case S16_TYPE:
+ case U16_TYPE:
+ case F16_TYPE:
+ g_size = 2; break;
+ case B32_TYPE:
+ case S32_TYPE:
+ case U32_TYPE:
+ case F32_TYPE:
+ g_size = 4; break;
+ case B64_TYPE:
+ case BB64_TYPE:
+ case S64_TYPE:
+ case U64_TYPE:
+ case F64_TYPE:
+ case FF64_TYPE:
+ g_size = 8; break;
+ case BB128_TYPE:
+ g_size = 16; break;
+ }
PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", g_ptx_token_decode[type_spec].c_str());
g_scalar_type.push_back( type_spec );
if ( g_scalar_type.size() > 1 ) {
parse_assert( (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP)
- || (g_opcode == TEX_OP),
- "only cvt, set, slct, and tex can have more than one type specifier.");
+ || (g_opcode == TEX_OP)|| (g_opcode==MMA_OP)|| (g_opcode == DP4A_OP),
+ "only cvt, set, slct, tex and dp4a can have more than one type specifier.");
}
g_scalar_type_spec = type_spec;
}
@@ -655,7 +716,11 @@ void add_option( int option )
PTX_PARSE_DPRINTF("add_option");
g_options.push_back( option );
}
-
+void add_wmma_option( int option )
+{
+ PTX_PARSE_DPRINTF("add_option");
+ g_wmma_options.push_back( option );
+}
void add_double_operand( const char *d1, const char *d2 )
{
//operands that access two variables.
@@ -711,6 +776,28 @@ void add_4vector_operand( const char *d1, const char *d2, const char *d3, const
if ( s4 == null_op ) s4 = NULL;
g_operands.push_back( operand_info(s1,s2,s3,s4) );
}
+void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4,const char *d5,const char *d6,const char *d7,const char *d8 )
+{
+ PTX_PARSE_DPRINTF("add_8vector_operand");
+ const symbol *s1 = g_current_symbol_table->lookup(d1);
+ const symbol *s2 = g_current_symbol_table->lookup(d2);
+ const symbol *s3 = g_current_symbol_table->lookup(d3);
+ const symbol *s4 = g_current_symbol_table->lookup(d4);
+ const symbol *s5 = g_current_symbol_table->lookup(d5);
+ const symbol *s6 = g_current_symbol_table->lookup(d6);
+ const symbol *s7 = g_current_symbol_table->lookup(d7);
+ const symbol *s8 = g_current_symbol_table->lookup(d8);
+ parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && s5 !=NULL && s6 !=NULL && s7 !=NULL && s8 !=NULL, "v4 component(s) missing declarations.");
+ const symbol *null_op = g_current_symbol_table->lookup("_");
+ if ( s2 == null_op ) s2 = NULL;
+ if ( s3 == null_op ) s3 = NULL;
+ if ( s4 == null_op ) s4 = NULL;
+ if ( s5 == null_op ) s5 = NULL;
+ if ( s6 == null_op ) s6 = NULL;
+ if ( s7 == null_op ) s7 = NULL;
+ if ( s8 == null_op ) s8 = NULL;
+ g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8) );
+}
void add_builtin_operand( int builtin, int dim_modifier )
{
diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h
index 13042e1..7b6e3a2 100644
--- a/src/cuda-sim/ptx_parser.h
+++ b/src/cuda-sim/ptx_parser.h
@@ -57,7 +57,9 @@ void add_1vector_operand( const char *d1 );
void add_2vector_operand( const char *d1, const char *d2 );
void add_3vector_operand( const char *d1, const char *d2, const char *d3 );
void add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 );
+void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ,const char *d5,const char *d6,const char *d7,const char *d8);
void add_option(int option );
+void add_wmma_option(int option );
void add_builtin_operand( int builtin, int dim_modifier );
void add_memory_operand( );
void add_literal_int( int value );
diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc
index a3e43aa..820287d 100644
--- a/src/cuda-sim/ptx_sim.cc
+++ b/src/cuda-sim/ptx_sim.cc
@@ -44,6 +44,7 @@ ptx_cta_info::ptx_cta_info( unsigned sm_idx )
m_sm_idx = sm_idx;
m_uid = g_ptx_cta_info_uid++;
+ m_bar_threads = 0;
}
void ptx_cta_info::add_thread( ptx_thread_info *thd )
@@ -128,6 +129,21 @@ unsigned ptx_cta_info::get_sm_idx() const
return m_sm_idx;
}
+unsigned ptx_cta_info::get_bar_threads() const
+{
+ return m_bar_threads;
+}
+
+void ptx_cta_info::inc_bar_threads()
+{
+ m_bar_threads++;
+}
+
+void ptx_cta_info::reset_bar_threads()
+{
+ m_bar_threads = 0;
+}
+
ptx_warp_info::ptx_warp_info()
{
reset_done_threads();
@@ -173,6 +189,7 @@ ptx_thread_info::ptx_thread_info( kernel_info_t &kernel )
m_last_memory_space = undefined_space;
m_branch_taken = 0;
m_shared_mem = NULL;
+ m_sstarr_mem = NULL;
m_warp_info = NULL;
m_cta_info = NULL;
m_local_mem = NULL;
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index e6eb02e..d226fbe 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -28,7 +28,7 @@
#define ptx_sim_h_INCLUDED
#include <stdlib.h>
-
+#include "half.h"
#include "../abstract_hardware_model.h"
#include "../tr1_hash_map.h"
@@ -42,6 +42,12 @@
#include "memory.h"
+#define GCC_VERSION (__GNUC__ * 10000 \
+ + __GNUC_MINOR__ * 100 \
+ + __GNUC_PATCHLEVEL__)
+
+
+
struct param_t {
const void *pdata;
int type;
@@ -53,6 +59,8 @@ struct param_t {
#include "memory.h"
+using half_float::half;
+
union ptx_reg_t {
ptx_reg_t() {
bits.ms = 0;
@@ -126,7 +134,12 @@ union ptx_reg_t {
unsigned short u16;
unsigned int u32;
unsigned long long u64;
- float f16;
+ //gcc 4.7.0
+ #if GCC_VERSION >= 40700
+ half f16;
+ #else
+ float f16;
+ #endif
float f32;
double f64;
struct {
@@ -158,8 +171,12 @@ public:
void register_thread_exit( ptx_thread_info *thd );
void register_deleted_thread( ptx_thread_info *thd );
unsigned get_sm_idx() const;
+ unsigned get_bar_threads() const;
+ void inc_bar_threads();
+ void reset_bar_threads();
private:
+ unsigned m_bar_threads;
unsigned long long m_uid;
unsigned m_sm_idx;
std::set<ptx_thread_info*> m_threads_in_cta;
@@ -169,7 +186,7 @@ private:
class ptx_warp_info {
public:
- ptx_warp_info();
+ ptx_warp_info(); // add get_core or something, or threads?
unsigned get_done_threads() const;
void inc_done_threads();
void reset_done_threads();
@@ -289,6 +306,8 @@ public:
const ptx_version &get_ptx_version() const;
void set_reg( const symbol *reg, const ptx_reg_t &value );
+ void print_reg_thread (char * fname);
+ void resume_reg_thread(char * fname, symbol_table * symtab);
ptx_reg_t get_reg( const symbol *reg );
ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag );
void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI );
@@ -299,6 +318,15 @@ public:
const ptx_reg_t &data2,
const ptx_reg_t &data3,
const ptx_reg_t &data4 );
+ void set_wmma_vector_operand_values( const operand_info &dst,
+ const ptx_reg_t &data1,
+ const ptx_reg_t &data2,
+ const ptx_reg_t &data3,
+ const ptx_reg_t &data4,
+ const ptx_reg_t &data5,
+ const ptx_reg_t &data6,
+ const ptx_reg_t &data7,
+ const ptx_reg_t &data8 );
function_info *func_info()
{
@@ -438,6 +466,7 @@ public:
memory_space_t m_last_memory_space;
dram_callback_t m_last_dram_callback;
memory_space *m_shared_mem;
+ memory_space *m_sstarr_mem;
memory_space *m_local_mem;
ptx_warp_info *m_warp_info;
ptx_cta_info *m_cta_info;