diff options
| -rw-r--r-- | src/abstract_hardware_model.cc | 288 | ||||
| -rw-r--r-- | src/abstract_hardware_model.h | 18 |
2 files changed, 204 insertions, 102 deletions
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 90f45d9..a7da2ff 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -95,12 +95,6 @@ address_type line_size_based_tag_func(new_addr_type address, new_addr_type line_ return address & ~(line_size-1); } -struct transaction_info { - std::bitset<4> chunks; // bitmask: 32-byte chunks accessed - mem_access_byte_mask_t bytes; - active_mask_t active; // threads in this transaction -}; - void warp_inst_t::clear_active( const active_mask_t &inactive ) { active_mask_t test = m_warp_active_mask; test &= inactive; @@ -263,102 +257,11 @@ void warp_inst_t::generate_mem_accesses() break; case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch == 13 ) { - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = 2; - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = 64; break; - case 4: case 8: case 16: segment_size = 128; break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map<new_addr_type,transaction_info> subwarp_transactions; - - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) { - if( !active(thread) ) - continue; - - // local memory can only be accessed in 4B chunks by one thread - unsigned data_size_coales = (space.get_type() == local_space || space.get_type() == param_space_local ) ? 4 : data_size; - unsigned num_accesses = (space.get_type() == local_space || space.get_type() == param_space_local ) ? data_size/4 : 1; - - for(unsigned access=0; access<num_accesses; access++) { - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access]; - unsigned block_address = line_size_based_tag_func(addr,segment_size); - unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access? - transaction_info &info = subwarp_transactions[block_address]; - - // can only write to one segment - assert(block_address == line_size_based_tag_func(addr+data_size_coales-1,segment_size)); - - info.chunks.set(chunk); - info.active.set(thread); - unsigned idx = (addr&127); - for( unsigned i=0; i < data_size_coales; i++ ) - info.bytes.set(idx+i); - } - } - - // step 2: reduce each transaction size, if possible - std::map< new_addr_type, transaction_info >::iterator t; - for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) { - new_addr_type addr = t->first; - assert( (addr & (segment_size-1)) == 0 ); - const transaction_info &info = t->second; - const std::bitset<4> &q = info.chunks; - assert( q.count() >= 1 ); - std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment) - - unsigned size=segment_size; - if( segment_size == 128 ) { - bool lower_half_used = q[0] || q[1]; - bool upper_half_used = q[2] || q[3]; - if( lower_half_used && !upper_half_used ) { - // only lower 64 bytes used - size = 64; - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else if ( (!lower_half_used) && upper_half_used ) { - // only upper 64 bytes used - addr = addr+64; - size = 64; - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } else { - assert(lower_half_used && upper_half_used); - } - } else if( segment_size == 64 ) { - // need to set halves - if( (addr % 128) == 0 ) { - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else { - assert( (addr % 128) == 64 ); - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } - } - if( size == 64 ) { - bool lower_half_used = h[0]; - bool upper_half_used = h[1]; - if( lower_half_used && !upper_half_used ) { - size = 32; - } else if ( (!lower_half_used) && upper_half_used ) { - addr = addr+32; - size = 32; - } else { - assert(lower_half_used && upper_half_used); - } - } - m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes) ); - } - } - + if(isatomic()) + memory_coalescing_arch_13_atomic(is_write, access_type); + else + memory_coalescing_arch_13(is_write, access_type); } else abort(); break; @@ -388,7 +291,188 @@ void warp_inst_t::generate_mem_accesses() ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); m_mem_accesses_created=true; -} +} + +void warp_inst_t::memory_coalescing_arch_13( bool is_write, mem_access_type access_type ) +{ + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = 2; + switch( data_size ) { + case 1: segment_size = 32; break; + case 2: segment_size = 64; break; + case 4: case 8: case 16: segment_size = 128; break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { + std::map<new_addr_type,transaction_info> subwarp_transactions; + + // step 1: find all transactions generated by this subwarp + for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) { + if( !active(thread) ) + continue; + + // local memory can only be accessed in 4B chunks by one thread + unsigned data_size_coales = (space.get_type() == local_space || space.get_type() == param_space_local ) ? 4 : data_size; + unsigned num_accesses = (space.get_type() == local_space || space.get_type() == param_space_local ) ? data_size/4 : 1; + + for(unsigned access=0; access<num_accesses; access++) { + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access]; + unsigned block_address = line_size_based_tag_func(addr,segment_size); + unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access? + transaction_info &info = subwarp_transactions[block_address]; + + // can only write to one segment + assert(block_address == line_size_based_tag_func(addr+data_size_coales-1,segment_size)); + + info.chunks.set(chunk); + info.active.set(thread); + unsigned idx = (addr&127); + for( unsigned i=0; i < data_size_coales; i++ ) + info.bytes.set(idx+i); + } + } + + // step 2: reduce each transaction size, if possible + std::map< new_addr_type, transaction_info >::iterator t; + for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) { + new_addr_type addr = t->first; + const transaction_info &info = t->second; + + memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size); + + } + } +} + +void warp_inst_t::memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type ) +{ + + assert(space.get_type() == global_space); // Atomics allowed only for global memory + + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = 2; + switch( data_size ) { + case 1: segment_size = 32; break; + case 2: segment_size = 64; break; + case 4: case 8: case 16: segment_size = 128; break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { + std::map<new_addr_type,std::list<transaction_info>> subwarp_transactions; // each block addr maps to a list of transactions + + // step 1: find all transactions generated by this subwarp + for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) { + if( !active(thread) ) + continue; + + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr,segment_size); + unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access? + + // can only write to one segment + assert(block_address == line_size_based_tag_func(addr+data_size-1,segment_size)); + + // Find a transaction that does not conflict with this thread's accesses + bool new_transaction = true; + std::list<transaction_info>::iterator it; + transaction_info* info; + for(it=subwarp_transactions[block_address].begin(); it!=subwarp_transactions[block_address].end(); it++) { + unsigned idx = (addr&127); + if(not it->test_bytes(idx,idx+data_size-1)) { + new_transaction = false; + info = &(*it); + break; + } + } + if(new_transaction) { + // Need a new transaction + subwarp_transactions[block_address].push_back(transaction_info()); + info = &subwarp_transactions[block_address].back(); + } + assert(info); + + info->chunks.set(chunk); + info->active.set(thread); + unsigned idx = (addr&127); + for( unsigned i=0; i < data_size; i++ ) { + assert(!info->bytes.test(idx+i)); + info->bytes.set(idx+i); + } + } + + // step 2: reduce each transaction size, if possible + std::map< new_addr_type, std::list<transaction_info> >::iterator t_list; + for( t_list=subwarp_transactions.begin(); t_list !=subwarp_transactions.end(); t_list++ ) { + // For each block addr + new_addr_type addr = t_list->first; + const std::list<transaction_info>& transaction_list = t_list->second; + + std::list<transaction_info>::const_iterator t; + for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { + // For each transaction + const transaction_info &info = *t; + memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size); + } + } + } +} + +void warp_inst_t::memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) +{ + assert( (addr & (segment_size-1)) == 0 ); + + const std::bitset<4> &q = info.chunks; + assert( q.count() >= 1 ); + std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment) + + unsigned size=segment_size; + if( segment_size == 128 ) { + bool lower_half_used = q[0] || q[1]; + bool upper_half_used = q[2] || q[3]; + if( lower_half_used && !upper_half_used ) { + // only lower 64 bytes used + size = 64; + if(q[0]) h.set(0); + if(q[1]) h.set(1); + } else if ( (!lower_half_used) && upper_half_used ) { + // only upper 64 bytes used + addr = addr+64; + size = 64; + if(q[2]) h.set(0); + if(q[3]) h.set(1); + } else { + assert(lower_half_used && upper_half_used); + } + } else if( segment_size == 64 ) { + // need to set halves + if( (addr % 128) == 0 ) { + if(q[0]) h.set(0); + if(q[1]) h.set(1); + } else { + assert( (addr % 128) == 64 ); + if(q[2]) h.set(0); + if(q[3]) h.set(1); + } + } + if( size == 64 ) { + bool lower_half_used = h[0]; + bool upper_half_used = h[1]; + if( lower_half_used && !upper_half_used ) { + size = 32; + } else if ( (!lower_half_used) && upper_half_used ) { + addr = addr+32; + size = 32; + } else { + assert(lower_half_used && upper_half_used); + } + } + m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes) ); +} + unsigned kernel_info_t::m_next_uid = 1; diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 4942110..8d4d009 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -687,7 +687,25 @@ public: for(unsigned i=0; i<num_addrs; i++) m_per_scalar_thread[n].memreqaddr[i] = addr[i]; } + + struct transaction_info { + std::bitset<4> chunks; // bitmask: 32-byte chunks accessed + mem_access_byte_mask_t bytes; + active_mask_t active; // threads in this transaction + + bool test_bytes(unsigned start_bit, unsigned end_bit) { + for( unsigned i=start_bit; i<=end_bit; i++ ) + if(bytes.test(i)) + return true; + return false; + } + }; + void generate_mem_accesses(); + void memory_coalescing_arch_13( bool is_write, mem_access_type access_type ); + void memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type ); + void memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); + void add_callback( unsigned lane_id, void (*function)(const class inst_t*, class ptx_thread_info*), const inst_t *inst, |