0.9756 correlation. Set L1T line size to 128 bytes... problem was

stalling to send four requests per warp into L1T tag lookup.

If L1T is really 32B blocks (as per Henry's paper), this suggests 
banking of L1T needs to be modeled. 

Other changes:
1. bug fix in memory access generation for texture/const cache access
2. adding back memory latency measurement for visualizer

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 7913]

13 files changed, 92 insertions, 118 deletions

diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index 7acd9bb..3f2883a 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -108,7 +108,7 @@ void warp_inst_t::generate_mem_accesses()
     }
 
     // Calculate memory accesses generated by this warp
-    new_addr_type cache_block_size = 32; // in bytes 
+    new_addr_type cache_block_size = 0; // in bytes 
 
     switch( space.get_type() ) {
     case shared_space: {
@@ -186,25 +186,9 @@ void warp_inst_t::generate_mem_accesses()
 
     case tex_space: 
         cache_block_size = m_config->gpgpu_cache_texl1_linesize;
-
+        break;
     case const_space:  case param_space_kernel:
-        cache_block_size = m_config->gpgpu_cache_constl1_linesize; {
-        mem_access_byte_mask_t byte_mask; 
-        std::map<new_addr_type,active_mask_t> accesses; // block address -> set of thread offsets in warp
-        std::map<new_addr_type,active_mask_t>::iterator a;
-        for( unsigned thread=0; thread < m_config->warp_size; thread++ ) {
-            if( !active(thread) ) 
-                continue;
-            new_addr_type addr = m_per_scalar_thread[thread].memreqaddr;
-            unsigned block_address = line_size_based_tag_func(addr,m_config->gpgpu_cache_texl1_linesize);
-            accesses[block_address].set(thread);
-            unsigned idx = addr-block_address; 
-            for( unsigned i=0; i < data_size; i++ ) 
-                byte_mask.set(idx+i);
-        }
-        for( a=accesses.begin(); a != accesses.end(); ++a ) 
-            m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second,byte_mask) );
-        }
+        cache_block_size = m_config->gpgpu_cache_constl1_linesize; 
         break;
 
     case global_space: case local_space: case param_space_local:
@@ -301,5 +285,24 @@ void warp_inst_t::generate_mem_accesses()
         abort();
     }
 
+    if( cache_block_size ) {
+        assert( m_accessq.empty() );
+        mem_access_byte_mask_t byte_mask; 
+        std::map<new_addr_type,active_mask_t> accesses; // block address -> set of thread offsets in warp
+        std::map<new_addr_type,active_mask_t>::iterator a;
+        for( unsigned thread=0; thread < m_config->warp_size; thread++ ) {
+            if( !active(thread) ) 
+                continue;
+            new_addr_type addr = m_per_scalar_thread[thread].memreqaddr;
+            unsigned block_address = line_size_based_tag_func(addr,cache_block_size);
+            accesses[block_address].set(thread);
+            unsigned idx = addr-block_address; 
+            for( unsigned i=0; i < data_size; i++ ) 
+                byte_mask.set(idx+i);
+        }
+        for( a=accesses.begin(); a != accesses.end(); ++a ) 
+            m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second,byte_mask) );
+    }
+
     m_mem_accesses_created=true;
 } 
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index bc73fbc..cdecc14 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -717,7 +717,7 @@ protected:
     bool m_per_scalar_thread_valid;
     std::vector<per_thread_info> m_per_scalar_thread;
     bool m_mem_accesses_created;
-    std::vector<mem_access_t> m_accessq;
+    std::list<mem_access_t> m_accessq;
 
     static unsigned sm_next_uid;
 };
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 1982218..b1893a0 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -142,21 +142,11 @@ void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* te
    printf("GPGPU-Sim PTX:   texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize());
    //first determine base Tx size for given linesize
    switch (m_function_model_config.get_texcache_linesize()) {
-   case 16:
-      Tx = 4;
-      break;
-   case 32:
-      Tx = 8;
-      break;
-   case 64:
-      Tx = 8;
-      break;
-   case 128:
-      Tx = 16;
-      break;
-   case 256:
-      Tx = 16;
-      break;
+   case 16:  Tx = 4; break;
+   case 32:  Tx = 8; break;
+   case 64:  Tx = 8; break;
+   case 128: Tx = 16; break;
+   case 256: Tx = 16; break;
    default:
       printf("GPGPU-Sim PTX:   Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize());
       assert(0);
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 2b6ce9c..ad2eb43 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -3745,11 +3745,8 @@ void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread )
       thread->m_last_effective_address = tex_array_index;
       break;
    case GEOM_MODIFIER_2D: 
-      x_block_coord = x;
-      x_block_coord = x_block_coord >> (texInfo->Tx_numbits + texInfo->texel_size_numbits);
-
-      y_block_coord = y;
-      y_block_coord = y_block_coord >> texInfo->Ty_numbits;
+      x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits);
+      y_block_coord = y >> texInfo->Ty_numbits;
 
       memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6;
 
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index c75b24e..de85b8f 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -657,7 +657,7 @@ public:
         for( int n=m_rob.size()-1; n>=0; n-- ) {
             unsigned index = (m_rob.next_pop_index() + n)%m_rob.capacity();
             const rob_entry &r = m_rob.peek(index);
-            fprintf(fp, "tex rob[%2d] : %s ", index, (r.m_ready?"ready  ":"pending") );
+            fprintf(fp, "tex rob[%3d] : %s ", index, (r.m_ready?"ready  ":"pending") );
             if( r.m_ready ) 
                 fprintf(fp,"@%6u", r.m_time );
             else 
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index dbae72c..fc93339 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -340,7 +340,7 @@ gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config )
 
     m_cluster = new simt_core_cluster*[m_shader_config->n_simt_clusters];
     for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) 
-        m_cluster[i] = new simt_core_cluster(this,i,m_shader_config,m_memory_config,m_shader_stats);
+        m_cluster[i] = new simt_core_cluster(this,i,m_shader_config,m_memory_config,m_shader_stats,m_memory_stats);
 
     m_memory_partition_unit = new memory_partition_unit*[m_memory_config->m_n_mem];
     for (unsigned i=0;i<m_memory_config->m_n_mem;i++) 
@@ -485,7 +485,7 @@ unsigned int gpgpu_sim::run_gpu_sim()
       if (m_config.gpu_deadlock_detect && gpu_deadlock) 
          break;
    }
-   m_memory_stats->memlatstat_lat_pw(m_config.num_shader(),m_shader_config->n_thread_per_shader,m_shader_config->warp_size);
+   m_memory_stats->memlatstat_lat_pw();
    gpu_tot_sim_cycle += gpu_sim_cycle;
    gpu_tot_sim_insn += gpu_sim_insn;
    
@@ -828,7 +828,7 @@ void gpgpu_sim::cycle()
                 (unsigned)days,(unsigned)hrs,(unsigned)minutes,(unsigned)sec,
                 ctime(&curr_time));
          fflush(stdout);
-         m_memory_stats->memlatstat_lat_pw(m_config.num_shader(),m_shader_config->n_thread_per_shader,m_shader_config->warp_size);
+         m_memory_stats->memlatstat_lat_pw();
          visualizer_printstat();
          if (m_config.gpgpu_runtime_stat && (m_config.gpu_runtime_stat_flag != 0) ) {
             if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_BW_STAT) {
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index 14f1f12..45b8eef 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -216,6 +216,16 @@ void memory_stats_t::print( FILE *fp )
     fprintf(fp,"L2_read_hit = %d\n", L2_read_hit);
 }
 
+void memory_stats_t::visualizer_print( gzFile visualizer_file )
+{
+   gzprintf(visualizer_file, "Ltwowritemiss: %d\n", L2_write_miss);
+   gzprintf(visualizer_file, "Ltwowritehit: %d\n",  L2_write_hit);
+   gzprintf(visualizer_file, "Ltworeadmiss: %d\n", L2_read_miss);
+   gzprintf(visualizer_file, "Ltworeadhit: %d\n", L2_read_hit);
+   if (num_mfs) 
+      gzprintf(visualizer_file, "averagemflatency: %lld\n", mf_total_lat/num_mfs);
+}
+
 void gpgpu_sim::L2c_print_cache_stat() const
 {
     unsigned i, j, k;
diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc
index aa0006c..f4c3bcc 100644
--- a/src/gpgpu-sim/mem_fetch.cc
+++ b/src/gpgpu-sim/mem_fetch.cc
@@ -131,7 +131,7 @@ void mem_fetch::print( FILE *fp, bool print_inst ) const
         fprintf(fp," <NULL mem_fetch pointer>\n");
         return;
     }
-    fprintf(fp,"  mf: uid=%6u, sid=%2u, partition=%u, ", m_request_uid, m_sid, m_raw_addr.chip );
+    fprintf(fp,"  mf: uid=%6u, sid%02u:w%02u, part=%u, ", m_request_uid, m_sid, m_wid, m_raw_addr.chip );
     m_access.print(fp);
     if( (unsigned)m_status < NUM_MEM_REQ_STAT ) 
        fprintf(fp," status = %s (%llu), ", Status_str[m_status], m_status_change );
diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc
index e1cbe2b..64510da 100644
--- a/src/gpgpu-sim/mem_latency_stat.cc
+++ b/src/gpgpu-sim/mem_latency_stat.cc
@@ -118,10 +118,6 @@ memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_conf
    memset(mf_lat_pw_table, 0, sizeof(unsigned)*32);
    mf_num_lat_pw = 0;
    max_warps = n_shader * (shader_config->n_thread_per_shader / shader_config->warp_size+1);
-   mf_num_lat_pw_perwarp = (unsigned *) calloc(max_warps, sizeof(unsigned int));
-   mf_tot_lat_pw_perwarp = (unsigned *) calloc(max_warps, sizeof(unsigned int));
-   mf_total_lat_perwarp = (unsigned long long int *) calloc(max_warps, sizeof(unsigned long long int));
-   num_mfs_perwarp = (unsigned *) calloc(max_warps, sizeof(unsigned int));
    mf_tot_lat_pw = 0; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window
    mf_total_lat = 0;
    num_mfs = 0;
@@ -176,16 +172,12 @@ memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_conf
    L2_L2todramlength = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int));
 }
 
-// recorder the total latency
-unsigned memory_stats_t::memlatstat_done(mem_fetch *mf, unsigned n_warp_per_shader )
+// record the total latency
+unsigned memory_stats_t::memlatstat_done(mem_fetch *mf )
 {
    unsigned mf_latency;
-   unsigned wid = mf->get_sid()*n_warp_per_shader + mf->get_wid();
-   assert(wid<max_warps);
    mf_latency = (gpu_sim_cycle+gpu_tot_sim_cycle) - mf->get_timestamp();
    mf_num_lat_pw++;
-   mf_num_lat_pw_perwarp[wid]++;
-   mf_tot_lat_pw_perwarp[wid] += mf_latency;
    mf_tot_lat_pw += mf_latency;
    unsigned idx = LOGB2(mf_latency);
    assert(idx<32);
@@ -197,10 +189,10 @@ unsigned memory_stats_t::memlatstat_done(mem_fetch *mf, unsigned n_warp_per_shad
    return mf_latency;
 }
 
-void memory_stats_t::memlatstat_read_done(mem_fetch *mf, unsigned n_warp_per_shader)
+void memory_stats_t::memlatstat_read_done(mem_fetch *mf)
 {
    if (m_memory_config->gpgpu_memlatency_stat) {
-      unsigned mf_latency = memlatstat_done(mf,n_warp_per_shader);
+      unsigned mf_latency = memlatstat_done(mf);
       if (mf_latency > mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk]) 
          mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = mf_latency;
       unsigned icnt2sh_latency;
@@ -244,9 +236,8 @@ void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf)
    }
 }
 
-void memory_stats_t::memlatstat_lat_pw( unsigned n_shader, unsigned n_thread_per_shader, unsigned warp_size )
+void memory_stats_t::memlatstat_lat_pw()
 {
-   unsigned i;
    if (mf_num_lat_pw && m_memory_config->gpgpu_memlatency_stat) {
       assert(mf_tot_lat_pw);
       mf_total_lat = mf_tot_lat_pw;
@@ -255,15 +246,6 @@ void memory_stats_t::memlatstat_lat_pw( unsigned n_shader, unsigned n_thread_per
       mf_tot_lat_pw = 0;
       mf_num_lat_pw = 0;
    }
-   for (i=0;i < ((n_shader * n_thread_per_shader / warp_size)+1); i++) {
-      assert(i<max_warps);
-      if (mf_num_lat_pw_perwarp[i] && m_memory_config->gpgpu_memlatency_stat) {
-         mf_total_lat_perwarp[i] += mf_tot_lat_pw_perwarp[i];
-         num_mfs_perwarp[i] += mf_num_lat_pw_perwarp[i];
-         mf_tot_lat_pw_perwarp[i] = 0;
-         mf_num_lat_pw_perwarp[i] = 0;
-      }
-   }
 }
 
 
diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h
index b7e4b64..87da046 100644
--- a/src/gpgpu-sim/mem_latency_stat.h
+++ b/src/gpgpu-sim/mem_latency_stat.h
@@ -69,6 +69,7 @@
 #define MEM_LATENCY_STAT_H
 
 #include <stdio.h>
+#include <zlib.h>
 
 class memory_stats_t {
 public:
@@ -76,14 +77,15 @@ public:
                    const struct shader_core_config *shader_config, 
                    const struct memory_config *mem_config );
 
-   unsigned memlatstat_done( class mem_fetch *mf, unsigned n_warp_per_shader );
-   void memlatstat_read_done( class mem_fetch *mf, unsigned n_warp_per_shader);
+   unsigned memlatstat_done( class mem_fetch *mf );
+   void memlatstat_read_done( class mem_fetch *mf );
    void memlatstat_dram_access( class mem_fetch *mf );
    void memlatstat_icnt2mem_pop( class mem_fetch *mf);
-   void memlatstat_lat_pw( unsigned n_shader, unsigned n_thread_per_shader, unsigned warp_size );
+   void memlatstat_lat_pw();
    void memlatstat_print(unsigned n_mem, unsigned gpu_mem_n_bk);
 
    void print( FILE *fp );
+   void visualizer_print( gzFile visualizer_file );
 
    unsigned m_n_shader;
 
@@ -115,10 +117,6 @@ public:
    unsigned int **totalbankaccesses; //bankaccesses[dram chip id][bank id]
    unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache
    unsigned int *position_of_mrq_chosen; //position of mrq in m_queue chosen 
-   unsigned *mf_num_lat_pw_perwarp;
-   unsigned *mf_tot_lat_pw_perwarp; //total latency summed up per window per warp. divide by mf_num_lat_pw_perwarp to obtain average latency Per Window
-   unsigned long long int *mf_total_lat_perwarp;
-   unsigned *num_mfs_perwarp;
    
    unsigned ***mem_access_type_stats; // dram access type classification
 
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 692eace..8adcf1b 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -480,36 +480,6 @@ void shader_core_stats::visualizer_print( gzFile visualizer_file )
    for (unsigned i=0;i<m_config->num_shader();i++) 
       gzprintf(visualizer_file, "%u ", m_n_diverge[i] );
    gzprintf(visualizer_file, "\n");
-
-/*
-   gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_All: ");
-   for (unsigned i=0;i<m_n_shader;i++) 
-      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1_windowed_cache_miss_rate(0));
-   gzprintf(visualizer_file, "\n");
-   gzprintf(visualizer_file, "CacheMissRate_TextureL1_All: ");
-   for (unsigned i=0;i<m_n_shader;i++) 
-      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1tex_windowed_cache_miss_rate(0));
-   gzprintf(visualizer_file, "\n");
-   gzprintf(visualizer_file, "CacheMissRate_ConstL1_All: ");
-   for (unsigned i=0;i<m_n_shader;i++) 
-      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1const_windowed_cache_miss_rate(0));
-   gzprintf(visualizer_file, "\n");
-   gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_noMgHt: ");
-   for (unsigned i=0;i<m_n_shader;i++) 
-      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1_windowed_cache_miss_rate(1));
-   gzprintf(visualizer_file, "\n");
-   gzprintf(visualizer_file, "CacheMissRate_TextureL1_noMgHt: ");
-   for (unsigned i=0;i<m_n_shader;i++) 
-      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1tex_windowed_cache_miss_rate(1));
-   gzprintf(visualizer_file, "\n");
-   gzprintf(visualizer_file, "CacheMissRate_ConstL1_noMgHt: ");
-   for (unsigned i=0;i<m_n_shader;i++) 
-      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1const_windowed_cache_miss_rate(1));
-   gzprintf(visualizer_file, "\n");
-   // reset for next interval
-   for (unsigned i=0;i<m_n_shader;i++) 
-      m_sc[i]->new_cache_window();
-*/
 }
 
 #define PROGRAM_MEM_START 0xF0000000 /* should be distinct from other memory spaces... 
@@ -1989,13 +1959,15 @@ simt_core_cluster::simt_core_cluster( class gpgpu_sim *gpu,
                                       unsigned cluster_id, 
                                       const struct shader_core_config *config, 
                                       const struct memory_config *mem_config,
-                                      shader_core_stats *stats )
+                                      shader_core_stats *stats, 
+                                      class memory_stats_t *mstats )
 {
     m_config = config;
     m_cta_issue_next_core=m_config->n_simt_cores_per_cluster-1; // this causes first launch to use hw cta 0
     m_cluster_id=cluster_id;
     m_gpu = gpu;
     m_stats = stats;
+    m_memory_stats = mstats;
     m_core = new shader_core_ctx*[ config->n_simt_cores_per_cluster ];
     for( unsigned i=0; i < config->n_simt_cores_per_cluster; i++ ) {
         unsigned sid = m_config->cid_to_sid(i,m_cluster_id);
@@ -2103,6 +2075,7 @@ void simt_core_cluster::icnt_cycle()
             // data response
             if( !m_core[cid]->ldst_unit_response_buffer_full() ) {
                 m_response_fifo.pop_front();
+                m_memory_stats->memlatstat_read_done(mf);
                 m_core[cid]->accept_ldst_unit_response(mf);
             }
         }
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 52a7e35..e85c210 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1166,7 +1166,8 @@ public:
                        unsigned cluster_id, 
                        const struct shader_core_config *config, 
                        const struct memory_config *mem_config,
-                       shader_core_stats *stats );
+                       shader_core_stats *stats,
+                       memory_stats_t *mstats );
 
     void core_cycle();
     void icnt_cycle();
@@ -1190,6 +1191,7 @@ private:
     gpgpu_sim *m_gpu;
     const shader_core_config *m_config;
     shader_core_stats *m_stats;
+    memory_stats_t *m_memory_stats;
     shader_core_ctx **m_core;
 
     unsigned m_cta_issue_next_core;
diff --git a/src/gpgpu-sim/visualizer.cc b/src/gpgpu-sim/visualizer.cc
index a6f836c..c2b1d16 100644
--- a/src/gpgpu-sim/visualizer.cc
+++ b/src/gpgpu-sim/visualizer.cc
@@ -95,20 +95,10 @@ void gpgpu_sim::visualizer_printstat()
    cflog_visualizer_gzprint(visualizer_file);
    shader_CTA_count_visualizer_gzprint(visualizer_file);
 
-   // per shader core cache miss rate 
-
    for (unsigned i=0;i<m_memory_config->m_n_mem;i++) 
       m_memory_partition_unit[i]->visualizer_print(visualizer_file);
    m_shader_stats->visualizer_print(visualizer_file);
-
-   gzprintf(visualizer_file, "Ltwowritemiss: %d\n", m_memory_stats->L2_write_miss);
-   gzprintf(visualizer_file, "Ltwowritehit: %d\n",  m_memory_stats->L2_write_hit);
-   gzprintf(visualizer_file, "Ltworeadmiss: %d\n", m_memory_stats->L2_read_miss);
-   gzprintf(visualizer_file, "Ltworeadhit: %d\n", m_memory_stats->L2_read_hit);
-
-   // latency stats
-   if (m_memory_stats->num_mfs) 
-      gzprintf(visualizer_file, "averagemflatency: %lld\n", m_memory_stats->mf_total_lat/m_memory_stats->num_mfs);
+   m_memory_stats->visualizer_print(visualizer_file);
 
    // other parameters for graphing
    gzprintf(visualizer_file, "globalcyclecount: %lld\n", gpu_sim_cycle);
@@ -118,6 +108,35 @@ void gpgpu_sim::visualizer_printstat()
    time_vector_print_interval2gzfile(visualizer_file);
 
    gzclose(visualizer_file);
+/*
+   gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_All: ");
+   for (unsigned i=0;i<m_n_shader;i++) 
+      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1_windowed_cache_miss_rate(0));
+   gzprintf(visualizer_file, "\n");
+   gzprintf(visualizer_file, "CacheMissRate_TextureL1_All: ");
+   for (unsigned i=0;i<m_n_shader;i++) 
+      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1tex_windowed_cache_miss_rate(0));
+   gzprintf(visualizer_file, "\n");
+   gzprintf(visualizer_file, "CacheMissRate_ConstL1_All: ");
+   for (unsigned i=0;i<m_n_shader;i++) 
+      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1const_windowed_cache_miss_rate(0));
+   gzprintf(visualizer_file, "\n");
+   gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_noMgHt: ");
+   for (unsigned i=0;i<m_n_shader;i++) 
+      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1_windowed_cache_miss_rate(1));
+   gzprintf(visualizer_file, "\n");
+   gzprintf(visualizer_file, "CacheMissRate_TextureL1_noMgHt: ");
+   for (unsigned i=0;i<m_n_shader;i++) 
+      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1tex_windowed_cache_miss_rate(1));
+   gzprintf(visualizer_file, "\n");
+   gzprintf(visualizer_file, "CacheMissRate_ConstL1_noMgHt: ");
+   for (unsigned i=0;i<m_n_shader;i++) 
+      gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1const_windowed_cache_miss_rate(1));
+   gzprintf(visualizer_file, "\n");
+   // reset for next interval
+   for (unsigned i=0;i<m_n_shader;i++) 
+      m_sc[i]->new_cache_window();
+*/
 }
 
 #include <list>