From 508c322551ccb5dfd2969344bcacde5ca759cf99 Mon Sep 17 00:00:00 2001
From: Inderpreet Singh <w6o6@ece.ubc.ca>
Date: Sun, 25 Dec 2011 18:17:07 -0800
Subject: Integrated in CL10086 from tm-test branch. Fix for Bug 119 -
 Incorrect coalescing of atomic accesses.

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 11226]
---
 src/abstract_hardware_model.cc | 288 ++++++++++++++++++++++++++---------------
 src/abstract_hardware_model.h  |  18 +++
 2 files changed, 204 insertions(+), 102 deletions(-)

(limited to 'src')

diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index 90f45d9..a7da2ff 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -95,12 +95,6 @@ address_type line_size_based_tag_func(new_addr_type address, new_addr_type line_
    return address & ~(line_size-1);
 }
 
-struct transaction_info {
-    std::bitset<4> chunks; // bitmask: 32-byte chunks accessed
-    mem_access_byte_mask_t bytes;
-    active_mask_t active; // threads in this transaction
-};
-
 void warp_inst_t::clear_active( const active_mask_t &inactive ) {
     active_mask_t test = m_warp_active_mask;
     test &= inactive;
@@ -263,102 +257,11 @@ void warp_inst_t::generate_mem_accesses()
         break;
 
     case global_space: case local_space: case param_space_local:
-         
         if( m_config->gpgpu_coalesce_arch == 13 ) {
-            // see the CUDA manual where it discusses coalescing rules before reading this
-            unsigned segment_size = 0;
-            unsigned warp_parts = 2;
-            switch( data_size ) {
-            case 1: segment_size = 32; break;
-            case 2: segment_size = 64; break;
-            case 4: case 8: case 16: segment_size = 128; break;
-            }
-            unsigned subwarp_size = m_config->warp_size / warp_parts;
-
-            for( unsigned subwarp=0; subwarp <  warp_parts; subwarp++ ) {
-                std::map<new_addr_type,transaction_info> subwarp_transactions;
-
-                // step 1: find all transactions generated by this subwarp
-                for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) {
-                    if( !active(thread) ) 
-                        continue;
-
-                    // local memory can only be accessed in 4B chunks by one thread
-                    unsigned data_size_coales = (space.get_type() == local_space || space.get_type() == param_space_local ) ? 4 : data_size;
-                    unsigned num_accesses = (space.get_type() == local_space || space.get_type() == param_space_local ) ? data_size/4 : 1;
-
-                    for(unsigned access=0; access<num_accesses; access++) {
-                        new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access];
-                        unsigned block_address = line_size_based_tag_func(addr,segment_size);
-                        unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access?
-                        transaction_info &info = subwarp_transactions[block_address];
-
-                        // can only write to one segment
-                        assert(block_address == line_size_based_tag_func(addr+data_size_coales-1,segment_size));
-
-                        info.chunks.set(chunk);
-                        info.active.set(thread);
-                        unsigned idx = (addr&127);
-                        for( unsigned i=0; i < data_size_coales; i++ )
-                            info.bytes.set(idx+i);
-                    }
-                }
-
-                // step 2: reduce each transaction size, if possible
-                std::map< new_addr_type, transaction_info >::iterator t;
-                for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) {
-                    new_addr_type addr = t->first;
-                    assert( (addr & (segment_size-1)) == 0 );
-                    const transaction_info &info = t->second;
-                    const std::bitset<4> &q = info.chunks;
-                    assert( q.count() >= 1 );
-                    std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment)
-
-                    unsigned size=segment_size;
-                    if( segment_size == 128 ) {
-                        bool lower_half_used = q[0] || q[1];
-                        bool upper_half_used = q[2] || q[3]; 
-                        if( lower_half_used && !upper_half_used ) {
-                            // only lower 64 bytes used
-                            size = 64;
-                            if(q[0]) h.set(0);
-                            if(q[1]) h.set(1);
-                        } else if ( (!lower_half_used) && upper_half_used ) {
-                            // only upper 64 bytes used
-                            addr = addr+64;
-                            size = 64;
-                            if(q[2]) h.set(0);
-                            if(q[3]) h.set(1);
-                        } else {
-                            assert(lower_half_used && upper_half_used);
-                        }
-                    } else if( segment_size == 64 ) {
-                        // need to set halves
-                        if( (addr % 128) == 0 ) {
-                            if(q[0]) h.set(0);
-                            if(q[1]) h.set(1);
-                        } else {
-                            assert( (addr % 128) == 64 );
-                            if(q[2]) h.set(0);
-                            if(q[3]) h.set(1);
-                        }
-                    }
-                    if( size == 64 ) {
-                        bool lower_half_used = h[0];
-                        bool upper_half_used = h[1]; 
-                        if( lower_half_used && !upper_half_used ) {
-                            size = 32;
-                        } else if ( (!lower_half_used) && upper_half_used ) {
-                            addr = addr+32;
-                            size = 32;
-                        } else {
-                            assert(lower_half_used && upper_half_used);
-                        }
-                    }
-                    m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes) );
-                }
-            }
-            
+           if(isatomic())
+               memory_coalescing_arch_13_atomic(is_write, access_type);
+           else
+               memory_coalescing_arch_13(is_write, access_type);
         } else abort();
 
         break;
@@ -388,7 +291,188 @@ void warp_inst_t::generate_mem_accesses()
 
     ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size );
     m_mem_accesses_created=true;
-} 
+}
+
+void warp_inst_t::memory_coalescing_arch_13( bool is_write, mem_access_type access_type )
+{
+    // see the CUDA manual where it discusses coalescing rules before reading this
+    unsigned segment_size = 0;
+    unsigned warp_parts = 2;
+    switch( data_size ) {
+    case 1: segment_size = 32; break;
+    case 2: segment_size = 64; break;
+    case 4: case 8: case 16: segment_size = 128; break;
+    }
+    unsigned subwarp_size = m_config->warp_size / warp_parts;
+
+    for( unsigned subwarp=0; subwarp <  warp_parts; subwarp++ ) {
+        std::map<new_addr_type,transaction_info> subwarp_transactions;
+
+        // step 1: find all transactions generated by this subwarp
+        for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) {
+            if( !active(thread) )
+                continue;
+
+            // local memory can only be accessed in 4B chunks by one thread
+            unsigned data_size_coales = (space.get_type() == local_space || space.get_type() == param_space_local ) ? 4 : data_size;
+            unsigned num_accesses = (space.get_type() == local_space || space.get_type() == param_space_local ) ? data_size/4 : 1;
+
+            for(unsigned access=0; access<num_accesses; access++) {
+                new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access];
+                unsigned block_address = line_size_based_tag_func(addr,segment_size);
+                unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access?
+                transaction_info &info = subwarp_transactions[block_address];
+
+                // can only write to one segment
+                assert(block_address == line_size_based_tag_func(addr+data_size_coales-1,segment_size));
+
+                info.chunks.set(chunk);
+                info.active.set(thread);
+                unsigned idx = (addr&127);
+                for( unsigned i=0; i < data_size_coales; i++ )
+                    info.bytes.set(idx+i);
+            }
+        }
+
+        // step 2: reduce each transaction size, if possible
+        std::map< new_addr_type, transaction_info >::iterator t;
+        for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) {
+            new_addr_type addr = t->first;
+            const transaction_info &info = t->second;
+
+            memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size);
+
+        }
+    }
+}
+
+void warp_inst_t::memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type )
+{
+
+   assert(space.get_type() == global_space); // Atomics allowed only for global memory
+
+   // see the CUDA manual where it discusses coalescing rules before reading this
+   unsigned segment_size = 0;
+   unsigned warp_parts = 2;
+   switch( data_size ) {
+   case 1: segment_size = 32; break;
+   case 2: segment_size = 64; break;
+   case 4: case 8: case 16: segment_size = 128; break;
+   }
+   unsigned subwarp_size = m_config->warp_size / warp_parts;
+
+   for( unsigned subwarp=0; subwarp <  warp_parts; subwarp++ ) {
+       std::map<new_addr_type,std::list<transaction_info>> subwarp_transactions; // each block addr maps to a list of transactions
+
+       // step 1: find all transactions generated by this subwarp
+       for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) {
+           if( !active(thread) )
+               continue;
+
+           new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0];
+           unsigned block_address = line_size_based_tag_func(addr,segment_size);
+           unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access?
+
+           // can only write to one segment
+           assert(block_address == line_size_based_tag_func(addr+data_size-1,segment_size));
+
+           // Find a transaction that does not conflict with this thread's accesses
+           bool new_transaction = true;
+           std::list<transaction_info>::iterator it;
+           transaction_info* info;
+           for(it=subwarp_transactions[block_address].begin(); it!=subwarp_transactions[block_address].end(); it++) {
+              unsigned idx = (addr&127);
+              if(not it->test_bytes(idx,idx+data_size-1)) {
+                 new_transaction = false;
+                 info = &(*it);
+                 break;
+              }
+           }
+           if(new_transaction) {
+              // Need a new transaction
+              subwarp_transactions[block_address].push_back(transaction_info());
+              info = &subwarp_transactions[block_address].back();
+           }
+           assert(info);
+
+           info->chunks.set(chunk);
+           info->active.set(thread);
+           unsigned idx = (addr&127);
+           for( unsigned i=0; i < data_size; i++ ) {
+               assert(!info->bytes.test(idx+i));
+               info->bytes.set(idx+i);
+           }
+       }
+
+       // step 2: reduce each transaction size, if possible
+       std::map< new_addr_type, std::list<transaction_info> >::iterator t_list;
+       for( t_list=subwarp_transactions.begin(); t_list !=subwarp_transactions.end(); t_list++ ) {
+           // For each block addr
+           new_addr_type addr = t_list->first;
+           const std::list<transaction_info>& transaction_list = t_list->second;
+
+           std::list<transaction_info>::const_iterator t;
+           for(t=transaction_list.begin(); t!=transaction_list.end(); t++) {
+               // For each transaction
+               const transaction_info &info = *t;
+               memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size);
+           }
+       }
+   }
+}
+
+void warp_inst_t::memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size )
+{
+   assert( (addr & (segment_size-1)) == 0 );
+
+   const std::bitset<4> &q = info.chunks;
+   assert( q.count() >= 1 );
+   std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment)
+
+   unsigned size=segment_size;
+   if( segment_size == 128 ) {
+       bool lower_half_used = q[0] || q[1];
+       bool upper_half_used = q[2] || q[3];
+       if( lower_half_used && !upper_half_used ) {
+           // only lower 64 bytes used
+           size = 64;
+           if(q[0]) h.set(0);
+           if(q[1]) h.set(1);
+       } else if ( (!lower_half_used) && upper_half_used ) {
+           // only upper 64 bytes used
+           addr = addr+64;
+           size = 64;
+           if(q[2]) h.set(0);
+           if(q[3]) h.set(1);
+       } else {
+           assert(lower_half_used && upper_half_used);
+       }
+   } else if( segment_size == 64 ) {
+       // need to set halves
+       if( (addr % 128) == 0 ) {
+           if(q[0]) h.set(0);
+           if(q[1]) h.set(1);
+       } else {
+           assert( (addr % 128) == 64 );
+           if(q[2]) h.set(0);
+           if(q[3]) h.set(1);
+       }
+   }
+   if( size == 64 ) {
+       bool lower_half_used = h[0];
+       bool upper_half_used = h[1];
+       if( lower_half_used && !upper_half_used ) {
+           size = 32;
+       } else if ( (!lower_half_used) && upper_half_used ) {
+           addr = addr+32;
+           size = 32;
+       } else {
+           assert(lower_half_used && upper_half_used);
+       }
+   }
+   m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes) );
+}
+
 
 unsigned kernel_info_t::m_next_uid = 1;
 
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index 4942110..8d4d009 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -687,7 +687,25 @@ public:
         for(unsigned i=0; i<num_addrs; i++)
             m_per_scalar_thread[n].memreqaddr[i] = addr[i];
     }
+
+    struct transaction_info {
+        std::bitset<4> chunks; // bitmask: 32-byte chunks accessed
+        mem_access_byte_mask_t bytes;
+        active_mask_t active; // threads in this transaction
+
+        bool test_bytes(unsigned start_bit, unsigned end_bit) {
+           for( unsigned i=start_bit; i<=end_bit; i++ )
+              if(bytes.test(i))
+                 return true;
+           return false;
+        }
+    };
+
     void generate_mem_accesses();
+    void memory_coalescing_arch_13( bool is_write, mem_access_type access_type );
+    void memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type );
+    void memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size );
+
     void add_callback( unsigned lane_id, 
                        void (*function)(const class inst_t*, class ptx_thread_info*),
                        const inst_t *inst, 
-- 
cgit v1.3