summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cuda-kernels/genericMatrixMultiply.cu314
-rw-r--r--src/cuda-sim/instructions.cc58
2 files changed, 346 insertions, 26 deletions
diff --git a/cuda-kernels/genericMatrixMultiply.cu b/cuda-kernels/genericMatrixMultiply.cu
new file mode 100644
index 0000000..95cf021
--- /dev/null
+++ b/cuda-kernels/genericMatrixMultiply.cu
@@ -0,0 +1,314 @@
+/* Copyright (c) 1993-2017, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+// Define some error checking macros.
+#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); }
+void cudaErrCheck_(cudaError_t stat, const char *file, int line) {
+ if (stat != cudaSuccess) {
+ fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line);
+ }
+}
+
+
+#include <mma.h>
+using namespace nvcuda;
+
+// Must be multiples of 16 for wmma code to work
+#define MATRIX_M (16)
+#define MATRIX_N (16)
+#define MATRIX_K (16)
+
+
+
+// The only dimensions currently supported by WMMA
+const int WMMA_M = 16;
+const int WMMA_N = 16;
+const int WMMA_K = 16;
+
+__global__ void wmma_example(half *a, half *b, float *c, int M, int N, int K, float alpha, float beta) {
+ // Leading dimensions. Packed with no transpositions.
+ int lda = M;
+ int ldb = K;
+ int ldc = M;
+
+ // Tile using a 2D grid
+ int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
+ int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
+
+ // Declare the fragments
+ wmma::fragment<wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, half, wmma::col_major> a_frag;
+ wmma::fragment<wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, half, wmma::col_major> b_frag;
+ wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> acc_frag;
+ wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> c_frag;
+
+ wmma::fill_fragment(acc_frag, 0.0f);
+
+ // Loop over k
+ for (int i = 0; i < K; i += WMMA_K) {
+ int aRow = warpM * WMMA_M;
+ int aCol = i;
+
+ int bRow = i;
+ int bCol = warpN * WMMA_N;
+
+ // Bounds checking
+ if (aRow < M && aCol < K && bRow < K && bCol < N) {
+ // Load the inputs
+ wmma::load_matrix_sync(a_frag, a + aRow + aCol * lda, lda);
+ wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
+
+ // Perform the matrix multiplication
+ wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
+
+ }
+ }
+
+ // Load in the current value of c, scale it by beta, and add this our result scaled by alpha
+ int cRow = warpM * WMMA_M;
+ int cCol = warpN * WMMA_N;
+
+ if (cRow < M && cCol < N) {
+ wmma::load_matrix_sync(c_frag, c + cRow + cCol * ldc, ldc, wmma::mem_col_major);
+
+
+ for(int i=0; i < c_frag.num_elements; i++) {
+ c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
+ }
+
+ // Store the output
+ wmma::store_matrix_sync(c + cRow + cCol * ldc, c_frag, ldc, wmma::mem_col_major);
+ }
+}
+
+__global__ void convertFp32ToFp16 (half *out, float *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n) {
+ out[idx] = in[idx];
+ }
+}
+
+int main(int argc, char* argv[]) {
+ float *a_fp32;
+ float *b_fp32;
+ half *a_fp16;
+ half *b_fp16;
+
+ float *c;
+ float *c_wmma;
+
+ float *d_host_wmma;
+ float *d_cal_host_wmma;
+ float *a_host_wmma;
+ float *b_host_wmma;
+ float *c_host_wmma;
+
+
+ cudaEvent_t startWMMA;
+ cudaEvent_t stopWMMA;
+
+
+ cudaErrCheck(cudaEventCreate(&startWMMA));
+ cudaErrCheck(cudaEventCreate(&stopWMMA));
+
+ // Use tensor cores
+ cudaErrCheck(cudaMalloc((void**)&a_fp32, MATRIX_M * MATRIX_K * sizeof(float)));
+ cudaErrCheck(cudaMalloc((void**)&b_fp32, MATRIX_K * MATRIX_N * sizeof(float)));
+ cudaErrCheck(cudaMalloc((void**)&a_fp16, MATRIX_M * MATRIX_K * sizeof(half)));
+ cudaErrCheck(cudaMalloc((void**)&b_fp16, MATRIX_K * MATRIX_N * sizeof(half)));
+
+ cudaErrCheck(cudaMalloc((void**)&c, MATRIX_M * MATRIX_N * sizeof(float)));
+ cudaErrCheck(cudaMalloc((void**)&c_wmma, MATRIX_M * MATRIX_N * sizeof(float)));
+
+ d_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float));
+ d_cal_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float));
+ c_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float));
+ a_host_wmma = (float*)malloc(MATRIX_M * MATRIX_K * sizeof(float));
+ b_host_wmma = (float*)malloc(MATRIX_K * MATRIX_N * sizeof(float));
+
+
+ printf("a\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_K;n++){
+ a_host_wmma[m*MATRIX_K+n]= (rand()%3);///3.0;
+ printf("%f ",a_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("b\n");
+ for(int m=0;m<MATRIX_K;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ b_host_wmma[m*MATRIX_N+n]=(rand()%3);///3.0;
+ printf("%f ",b_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("c\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ c_host_wmma[m*MATRIX_N+n]= (rand()%3);///3.0;
+ printf("%f ",c_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+
+ cudaErrCheck(cudaMemcpy(a_fp32,a_host_wmma, MATRIX_M * MATRIX_K * sizeof(float), cudaMemcpyHostToDevice));
+ cudaErrCheck(cudaMemcpy(b_fp32,b_host_wmma, MATRIX_K * MATRIX_N * sizeof(float), cudaMemcpyHostToDevice));
+
+ convertFp32ToFp16 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (a_fp16, a_fp32, MATRIX_M * MATRIX_K);
+ convertFp32ToFp16 <<< (MATRIX_K * MATRIX_N + 255) / 256, 256 >>> (b_fp16, b_fp32, MATRIX_K * MATRIX_N);
+
+ cudaErrCheck(cudaMemcpy(c, c_host_wmma, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyHostToDevice));
+ cudaErrCheck(cudaMemcpy(c_wmma, c, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToDevice));
+
+ float alpha = 1.0f;
+ float beta = 1.0f;
+
+
+ printf("\nM = %d, N = %d, K = %d. alpha = %f, beta = %f\n\n", MATRIX_M, MATRIX_N, MATRIX_K, alpha, beta);
+
+ // First: using WMMA
+ dim3 gridDim;
+ dim3 blockDim;
+
+ // blockDim.x must be a multple of warpSize
+ // 128x4 means we have 16 warps and a block computes a 64x64 output tile
+ blockDim.x = 64;
+ blockDim.y = 2;
+
+ gridDim.x = (MATRIX_M + (WMMA_M * blockDim.x / 32 - 1)) / (WMMA_M * blockDim.x / 32);
+ gridDim.y = (MATRIX_N + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
+ printf("GRID:X=%d,Y=%d\n",gridDim.x,gridDim.y);
+ printf("BLOCK:X=%d,Y=%d\n",blockDim.x,blockDim.y);
+
+ printf("Running with wmma...\n");
+ cudaErrCheck(cudaEventRecord(startWMMA));
+ wmma_example <<< gridDim, blockDim >>> (a_fp16, b_fp16, c_wmma, MATRIX_M, MATRIX_N, MATRIX_K, alpha, beta);
+ cudaErrCheck(cudaEventRecord(stopWMMA));
+ cudaErrCheck(cudaEventSynchronize(stopWMMA));
+
+ printf("\nChecking results...\n");
+ cudaErrCheck(cudaMemcpy(d_host_wmma, c_wmma, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost));
+
+ printf("wmma:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%f ",d_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("wmma:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%f ",d_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("wmma:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%f ",d_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("wmma:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%f ",d_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ d_cal_host_wmma[n*MATRIX_N+m]=0;
+ for(int k=0;k<MATRIX_K;k++){
+ d_cal_host_wmma[n*MATRIX_N+m]+= a_host_wmma[k*MATRIX_K+m]*b_host_wmma[n*MATRIX_K+k];
+ }
+ d_cal_host_wmma[n*MATRIX_N+m]+=c_host_wmma[n*MATRIX_N+m];
+ }
+ }
+ printf("cal:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%.2f ",d_cal_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("wmma:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%.2f ",d_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ int suc=1;
+ float relative_error;
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ relative_error=100*abs(d_cal_host_wmma[m*MATRIX_N+n]-d_host_wmma[m*MATRIX_N+n])/d_host_wmma[m*MATRIX_N+n];
+ printf("relative_error=%f\n",relative_error);
+ if((int)relative_error>1)
+ {
+ printf("ERROR:\n");
+ suc=0;
+ printf("ROW=%d,COL=%d:cpu=%f,gpgpusim=%f\n",m,n,d_cal_host_wmma[m*MATRIX_N+n],d_host_wmma[m*MATRIX_N+n]);
+ }
+ }
+ }
+ if(suc==1)
+ printf("COMPLETED_SUCCESSFULLY\n");
+
+ //int errors = 0;
+ //for (int i = 0; i < MATRIX_M * MATRIX_N; i++) {
+ // float v1 = c_host_wmma[i];
+ // float v2 = c_host_cublas[i];
+ // if (v1 / v2 > 1.0001 || v2 / v1 > 1.0001 || abs(v1 - v2) > 1e-5) {
+ // errors++;
+ // if (errors < 10) printf("%f %f\n", v1, v2);
+ // }
+ //}
+
+ float wmmaTime;
+ cudaErrCheck(cudaEventElapsedTime(&wmmaTime, startWMMA, stopWMMA));
+ printf("wmma took %fms\n", wmmaTime);
+
+ cudaErrCheck(cudaEventDestroy(startWMMA));
+ cudaErrCheck(cudaEventDestroy(stopWMMA));
+ cudaErrCheck(cudaFree(a_fp32));
+ cudaErrCheck(cudaFree(b_fp32));
+ cudaErrCheck(cudaFree(a_fp16));
+ cudaErrCheck(cudaFree(b_fp16));
+ cudaErrCheck(cudaFree(c));
+ cudaErrCheck(cudaFree(c_wmma));
+ free(d_host_wmma);
+ free(c_host_wmma);
+ cudaErrCheck(cudaDeviceReset());
+ return 0;
+}
+
+
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 8b66cd0..fc47d9a 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -57,7 +57,7 @@ const char *g_opcode_string[NUM_OPCODES] = {
#undef OP_W_DEF
};
//Using profiled information::check the TensorCoreMatrixArrangement.xls for details
-unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type){
+unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){
unsigned offset;
unsigned load_a_row[8]={0,128,0,128,64,192,64,192};
@@ -132,24 +132,27 @@ unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout
abort();
}
-
+ offset = (offset/16)*stride+offset%16;
return offset;
}
-int acc_float_offset(int index,int wmma_layout){
+int acc_float_offset(int index,int wmma_layout,int stride){
int c_row_offset[]={0,1,32,33,4,5,36,37};
int c_col_offset[]={0,16,2,18,64,80,66,82};
-
+ int offset;
+
+
if(wmma_layout==ROW)
- return c_row_offset[index];
+ offset=c_row_offset[index];
else if(wmma_layout==COL)
- return c_col_offset[index];
+ offset=c_col_offset[index];
else{
printf("wrong layout");
abort();
}
-
+ offset = (offset/16)*stride+offset%16;
+ return offset;
}
void inst_not_implemented( const ptx_instruction * pI ) ;
@@ -1605,7 +1608,7 @@ unsigned trunc(unsigned num, unsigned precision) {
}
return num;
}
-void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int &row,int &col,int &assg_offset){
+void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){
int offset;
int c_row_offset[]={0,8,0,8,4,12,4,12};
int c_col_offset[]={0,0,8,8,0,0,8,8};
@@ -1614,7 +1617,7 @@ void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int &ro
int c_inside_row_offset[]={0,0,2,2,0,0,2,2};
int c_inside_col_offset[]={0,1,0,1,4,5,4,5};
- offset=thread_group_offset(thread,wmma_type,wmma_layout,type);
+ offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride);
if(wmma_type==LOAD_A){
if(wmma_layout==ROW){
@@ -1623,6 +1626,7 @@ void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int &ro
else{
offset+=64*(index/4)+index%4+128*((thread%16)/8);
}
+ offset=(offset/16)*stride+offset%16;
assg_offset=index+8*((thread%16)/8);
}
else if(wmma_type==LOAD_B){
@@ -1632,6 +1636,7 @@ void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int &ro
else{
offset+=index+8*((thread%16)/8);
}
+ offset=(offset/16)*stride+offset%16;
assg_offset=index+8*((thread%16)/8);
}
else if( wmma_type==LOAD_C){
@@ -1668,7 +1673,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
ptx_reg_t matrix_d[16][16];
ptx_reg_t src_data;
ptx_thread_info *thread;
-
+ int stride;
unsigned wmma_type = pI->get_wmma_type();
unsigned a_layout = pI->get_wmma_layout(0);
unsigned b_layout = pI->get_wmma_layout(1);
@@ -1724,21 +1729,21 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
switch(i) {
case 1 ://operand 1
for(k=0;k<8;k++){
- mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,row,col,offset);
+ mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset);
printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset);
matrix_a[row][col]=nw_v[offset];
}
break;
case 2 ://operand 2
for(k=0;k<8;k++){
- mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,row,col,offset);
+ mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset);
printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset);
matrix_b[row][col]=nw_v[offset];
}
break;
case 3 ://operand 3
for(k=0;k<8;k++){
- mapping(thrd,LOAD_C,ROW,type2,k,row,col,offset);
+ mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset);
printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset);
if(type2!=F16_TYPE){
matrix_c[row][col]=v[offset];
@@ -1842,7 +1847,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
int row_t[8];
int col_t[8];
for(k=0;k<8;k++){
- mapping(thrd,LOAD_C,ROW,type,k,row_t[k],col_t[k],offset);
+ mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset);
printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]);
}
thread = core->get_thread_info()[tid+thrd];
@@ -2909,7 +2914,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
unsigned type = pI->get_type();
unsigned wmma_type = pI->get_wmma_type();
unsigned wmma_layout = pI->get_wmma_layout(0);
-
+ int stride;
for (thrd=0; thrd < core->get_warp_size(); thrd++) {
thread = core->get_thread_info()[tid+thrd];
odd=thrd%2;
@@ -2920,7 +2925,8 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
unsigned nelem = src_a.get_vect_nelem();
ptx_reg_t* v= new ptx_reg_t[8];
thread->get_vector_operand_values( src_a, v, nelem );
-
+ stride=src2_data.u32;
+
memory_space_t space = pI->get_space();
memory_space *mem = NULL;
@@ -2930,7 +2936,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
type_info_key::type_decode(type,size,t);
printf("mma_st: thrd=%d,addr=%d, fp(size=%d), stride=%d\n",thrd,addr_reg.u32,size,src2_data.u32);
- addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type)*size/8;
+ addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8;
ptx_reg_t nw_v[8];
for(k=0;k<8;k++){
@@ -2942,7 +2948,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
for(k=0;k<8;k++){
if(type==F32_TYPE){
- mem->write(new_addr+4*acc_float_offset(k,wmma_layout),size/8,&v[k].s64,thread,pI);
+ mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI);
printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64);
float temp;
@@ -2959,7 +2965,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
if(wmma_layout==ROW)
mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI);
else if(wmma_layout==COL)
- mem->write(new_addr+k*32,size/8,&nw_v[k].s64,thread,pI);
+ mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI);
printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64);
}
}
@@ -2982,14 +2988,14 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
unsigned wmma_type = pI->get_wmma_type();
unsigned wmma_layout = pI->get_wmma_layout(0);
int tid = inst.warp_id_func()*core->get_warp_size();
- int thrd;
+ int thrd,stride;
ptx_thread_info *thread;
for (thrd=0; thrd < core->get_warp_size(); thrd++){
thread = core->get_thread_info()[tid+thrd];
ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1);
ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1);
-
+ stride=src2_data.u32;
memory_space_t space = pI->get_space();
memory_space *mem = NULL;
@@ -2999,14 +3005,14 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
ptx_reg_t data[16];
printf("mma_ld: thrd=%d,addr=%d, fpsize=%d, stride=%d\n",thrd,src1_data.u32,size,src2_data.u32);
- addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type)*size/8;
+ addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8;
if(wmma_type==LOAD_A){
for(i=0;i<16;i++){
if(wmma_layout==ROW)
mem->read(new_addr+2*i,size/8,&data[i].s64);
else if(wmma_layout==COL){
- mem->read(new_addr+2*(i%4)+128*(i/4),size/8,&data[i].s64);
+ mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64);
}
else{
printf("mma_ld:wrong_layout_type\n");
@@ -3019,7 +3025,7 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
if(wmma_layout==COL)
mem->read(new_addr+2*i,size/8,&data[i].s64);
else if(wmma_layout==ROW){
- mem->read(new_addr+2*(i%4)+128*(i/4),size/8,&data[i].s64);
+ mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64);
}
else{
printf("mma_ld:wrong_layout_type\n");
@@ -3033,14 +3039,14 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
if(wmma_layout==ROW)
mem->read(new_addr+2*i,size/8,&data[i].s64);
else if(wmma_layout==COL)
- mem->read(new_addr+32*i,size/8,&data[i].s64);
+ mem->read(new_addr+2*stride*i,size/8,&data[i].s64);
else{
printf("mma_ld:wrong_type\n");
abort();
}
}
else if(type==F32_TYPE){
- mem->read(new_addr+4*acc_float_offset(i,wmma_layout),size/8,&data[i].s64);
+ mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64);
}
else{
printf("wrong type");
generated by cgit v1.3 (git 2.53.0) at 2026-07-11 08:32:06 +0000