summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cuda-kernels/genericMatrixMultiply.cu48
-rw-r--r--cuda-kernels/genericMatrixMultiplyRow.cu289
-rw-r--r--cuda-kernels/v4p_genericMatrixMultiply.cu384
-rw-r--r--src/cuda-sim/instructions.cc71
4 files changed, 731 insertions, 61 deletions
diff --git a/cuda-kernels/genericMatrixMultiply.cu b/cuda-kernels/genericMatrixMultiply.cu
index 95cf021..6a5d33f 100644
--- a/cuda-kernels/genericMatrixMultiply.cu
+++ b/cuda-kernels/genericMatrixMultiply.cu
@@ -40,9 +40,9 @@ void cudaErrCheck_(cudaError_t stat, const char *file, int line) {
using namespace nvcuda;
// Must be multiples of 16 for wmma code to work
-#define MATRIX_M (16)
-#define MATRIX_N (16)
-#define MATRIX_K (16)
+#define MATRIX_M (32)
+#define MATRIX_N (32)
+#define MATRIX_K (32)
@@ -152,27 +152,27 @@ int main(int argc, char* argv[]) {
b_host_wmma = (float*)malloc(MATRIX_K * MATRIX_N * sizeof(float));
- printf("a\n");
+ printf("INITIAL_MATRIX_A\n");
for(int m=0;m<MATRIX_M;m++){
for(int n=0;n<MATRIX_K;n++){
a_host_wmma[m*MATRIX_K+n]= (rand()%3);///3.0;
- printf("%f ",a_host_wmma[m*MATRIX_K+n]);
+ printf("%.2f ",a_host_wmma[m*MATRIX_K+n]);
}
printf("\n");
}
- printf("b\n");
+ printf("INITIAL_MATRIX_B\n");
for(int m=0;m<MATRIX_K;m++){
for(int n=0;n<MATRIX_N;n++){
b_host_wmma[m*MATRIX_N+n]=(rand()%3);///3.0;
- printf("%f ",b_host_wmma[m*MATRIX_K+n]);
+ printf("%.2f ",b_host_wmma[m*MATRIX_K+n]);
}
printf("\n");
}
- printf("c\n");
+ printf("INITIAL_MATRIX_C\n");
for(int m=0;m<MATRIX_M;m++){
for(int n=0;n<MATRIX_N;n++){
c_host_wmma[m*MATRIX_N+n]= (rand()%3);///3.0;
- printf("%f ",c_host_wmma[m*MATRIX_K+n]);
+ printf("%.2f ",c_host_wmma[m*MATRIX_K+n]);
}
printf("\n");
}
@@ -215,34 +215,8 @@ int main(int argc, char* argv[]) {
printf("\nChecking results...\n");
cudaErrCheck(cudaMemcpy(d_host_wmma, c_wmma, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost));
- printf("wmma:d\n");
- for(int m=0;m<MATRIX_M;m++){
- for(int n=0;n<MATRIX_N;n++){
- printf("%f ",d_host_wmma[m*MATRIX_K+n]);
- }
- printf("\n");
- }
- printf("wmma:d\n");
- for(int m=0;m<MATRIX_M;m++){
- for(int n=0;n<MATRIX_N;n++){
- printf("%f ",d_host_wmma[m*MATRIX_K+n]);
- }
- printf("\n");
- }
- printf("wmma:d\n");
- for(int m=0;m<MATRIX_M;m++){
- for(int n=0;n<MATRIX_N;n++){
- printf("%f ",d_host_wmma[m*MATRIX_K+n]);
- }
- printf("\n");
- }
- printf("wmma:d\n");
- for(int m=0;m<MATRIX_M;m++){
- for(int n=0;n<MATRIX_N;n++){
- printf("%f ",d_host_wmma[m*MATRIX_K+n]);
- }
- printf("\n");
- }
+ int t=200000000;
+ while(t-->0);
for(int m=0;m<MATRIX_M;m++){
for(int n=0;n<MATRIX_N;n++){
d_cal_host_wmma[n*MATRIX_N+m]=0;
diff --git a/cuda-kernels/genericMatrixMultiplyRow.cu b/cuda-kernels/genericMatrixMultiplyRow.cu
new file mode 100644
index 0000000..6194492
--- /dev/null
+++ b/cuda-kernels/genericMatrixMultiplyRow.cu
@@ -0,0 +1,289 @@
+/* Copyright (c) 1993-2017, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of NVIDIA CORPORATION nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+// Define some error checking macros.
+#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); }
+void cudaErrCheck_(cudaError_t stat, const char *file, int line) {
+ if (stat != cudaSuccess) {
+ fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line);
+ }
+}
+
+
+#include <mma.h>
+using namespace nvcuda;
+
+// Must be multiples of 16 for wmma code to work
+#define MATRIX_M (32)
+#define MATRIX_N (32)
+#define MATRIX_K (32)
+
+
+
+// The only dimensions currently supported by WMMA
+const int WMMA_M = 16;
+const int WMMA_N = 16;
+const int WMMA_K = 16;
+
+__global__ void wmma_example(half *a, half *b, float *c, int M, int N, int K, float alpha, float beta) {
+ // Leading dimensions. Packed with no transpositions.
+ int lda = M;
+ int ldb = K;
+ int ldc = M;
+
+ // Tile using a 2D grid
+ int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
+ int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
+
+ // Declare the fragments
+ wmma::fragment<wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, half, wmma::row_major> a_frag;
+ wmma::fragment<wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, half, wmma::row_major> b_frag;
+ wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> acc_frag;
+ wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> c_frag;
+
+ wmma::fill_fragment(acc_frag, 0.0f);
+
+ // Loop over k
+ for (int i = 0; i < K; i += WMMA_K) {
+ int aRow = warpM * WMMA_M;
+ int aCol = i;
+
+ int bRow = i;
+ int bCol = warpN * WMMA_N;
+
+ // Bounds checking
+ if (aRow < M && aCol < K && bRow < K && bCol < N) {
+ // Load the inputs
+ wmma::load_matrix_sync(a_frag, a + aRow * lda+ aCol , lda);
+ wmma::load_matrix_sync(b_frag, b + bRow * ldb+ bCol , ldb);
+
+ // Perform the matrix multiplication
+ wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
+
+ }
+ }
+
+ // Load in the current value of c, scale it by beta, and add this our result scaled by alpha
+ int cRow = warpM * WMMA_M;
+ int cCol = warpN * WMMA_N;
+
+ if (cRow < M && cCol < N) {
+ wmma::load_matrix_sync(c_frag, c + cRow*ldc + cCol , ldc, wmma::mem_row_major);
+
+
+ for(int i=0; i < c_frag.num_elements; i++) {
+ c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
+ }
+
+ // Store the output
+ wmma::store_matrix_sync(c + cRow *ldc + cCol , c_frag, ldc, wmma::mem_row_major);
+ }
+}
+
+__global__ void convertFp32ToFp16 (half *out, float *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n) {
+ out[idx] = in[idx];
+ }
+}
+
+int main(int argc, char* argv[]) {
+ float *a_fp32;
+ float *b_fp32;
+ half *a_fp16;
+ half *b_fp16;
+
+ float *c;
+ float *c_wmma;
+
+ float *d_host_wmma;
+ float *d_cal_host_wmma;
+ float *a_host_wmma;
+ float *b_host_wmma;
+ float *c_host_wmma;
+
+
+ cudaEvent_t startWMMA;
+ cudaEvent_t stopWMMA;
+
+
+ cudaErrCheck(cudaEventCreate(&startWMMA));
+ cudaErrCheck(cudaEventCreate(&stopWMMA));
+
+ // Use tensor cores
+ cudaErrCheck(cudaMalloc((void**)&a_fp32, MATRIX_M * MATRIX_K * sizeof(float)));
+ cudaErrCheck(cudaMalloc((void**)&b_fp32, MATRIX_K * MATRIX_N * sizeof(float)));
+ cudaErrCheck(cudaMalloc((void**)&a_fp16, MATRIX_M * MATRIX_K * sizeof(half)));
+ cudaErrCheck(cudaMalloc((void**)&b_fp16, MATRIX_K * MATRIX_N * sizeof(half)));
+
+ cudaErrCheck(cudaMalloc((void**)&c, MATRIX_M * MATRIX_N * sizeof(float)));
+ cudaErrCheck(cudaMalloc((void**)&c_wmma, MATRIX_M * MATRIX_N * sizeof(float)));
+
+ d_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float));
+ d_cal_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float));
+ c_host_wmma = (float*)malloc(MATRIX_M * MATRIX_N * sizeof(float));
+ a_host_wmma = (float*)malloc(MATRIX_M * MATRIX_K * sizeof(float));
+ b_host_wmma = (float*)malloc(MATRIX_K * MATRIX_N * sizeof(float));
+
+
+ printf("INITIAL_MATRIX_A\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_K;n++){
+ a_host_wmma[m*MATRIX_K+n]= (rand()%3);///3.0;
+ printf("%.2f ",a_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("INITIAL_MATRIX_B\n");
+ for(int m=0;m<MATRIX_K;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ b_host_wmma[m*MATRIX_N+n]=(rand()%3);///3.0;
+ printf("%.2f ",b_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("INITIAL_MATRIX_C\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ c_host_wmma[m*MATRIX_N+n]= (rand()%3);///3.0;
+ printf("%.2f ",c_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+
+ cudaErrCheck(cudaMemcpy(a_fp32,a_host_wmma, MATRIX_M * MATRIX_K * sizeof(float), cudaMemcpyHostToDevice));
+ cudaErrCheck(cudaMemcpy(b_fp32,b_host_wmma, MATRIX_K * MATRIX_N * sizeof(float), cudaMemcpyHostToDevice));
+
+ convertFp32ToFp16 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (a_fp16, a_fp32, MATRIX_M * MATRIX_K);
+ convertFp32ToFp16 <<< (MATRIX_K * MATRIX_N + 255) / 256, 256 >>> (b_fp16, b_fp32, MATRIX_K * MATRIX_N);
+
+ cudaErrCheck(cudaMemcpy(c, c_host_wmma, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyHostToDevice));
+ cudaErrCheck(cudaMemcpy(c_wmma, c, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToDevice));
+
+ float alpha = 1.0f;
+ float beta = 1.0f;
+
+
+ printf("\nM = %d, N = %d, K = %d. alpha = %f, beta = %f\n\n", MATRIX_M, MATRIX_N, MATRIX_K, alpha, beta);
+
+ // First: using WMMA
+ dim3 gridDim;
+ dim3 blockDim;
+
+ // blockDim.x must be a multple of warpSize
+ // 128x4 means we have 16 warps and a block computes a 64x64 output tile
+ blockDim.x = 64;
+ blockDim.y = 2;
+
+ gridDim.x = (MATRIX_M + (WMMA_M * blockDim.x / 32 - 1)) / (WMMA_M * blockDim.x / 32);
+ gridDim.y = (MATRIX_N + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
+ printf("GRID:X=%d,Y=%d\n",gridDim.x,gridDim.y);
+ printf("BLOCK:X=%d,Y=%d\n",blockDim.x,blockDim.y);
+
+ printf("Running with wmma...\n");
+ cudaErrCheck(cudaEventRecord(startWMMA));
+ wmma_example <<< gridDim, blockDim >>> (a_fp16, b_fp16, c_wmma, MATRIX_M, MATRIX_N, MATRIX_K, alpha, beta);
+ cudaErrCheck(cudaEventRecord(stopWMMA));
+ cudaErrCheck(cudaEventSynchronize(stopWMMA));
+
+ printf("\nChecking results...\n");
+ cudaErrCheck(cudaMemcpy(d_host_wmma, c_wmma, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost));
+
+ int t=200000000;
+ while(t-->0);
+
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ d_cal_host_wmma[m*MATRIX_N+n]=0;
+ for(int k=0;k<MATRIX_K;k++){
+ d_cal_host_wmma[m*MATRIX_N+n]+= a_host_wmma[m*MATRIX_K+k]*b_host_wmma[k*MATRIX_K+n];
+ }
+ d_cal_host_wmma[m*MATRIX_N+n]+=c_host_wmma[m*MATRIX_N+n];
+ }
+ }
+ printf("cal:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%.2f ",d_cal_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ printf("wmma:d\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%.2f ",d_host_wmma[m*MATRIX_K+n]);
+ }
+ printf("\n");
+ }
+ int suc=1;
+ float relative_error;
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ relative_error=100*abs(d_cal_host_wmma[m*MATRIX_N+n]-d_host_wmma[m*MATRIX_N+n])/d_host_wmma[m*MATRIX_N+n];
+ printf("relative_error=%f\n",relative_error);
+ if((int)relative_error>1)
+ {
+ printf("ERROR:\n");
+ suc=0;
+ printf("ROW=%d,COL=%d:cpu=%f,gpgpusim=%f\n",m,n,d_cal_host_wmma[m*MATRIX_N+n],d_host_wmma[m*MATRIX_N+n]);
+ }
+ }
+ }
+ if(suc==1)
+ printf("COMPLETED_SUCCESSFULLY\n");
+
+ //int errors = 0;
+ //for (int i = 0; i < MATRIX_M * MATRIX_N; i++) {
+ // float v1 = c_host_wmma[i];
+ // float v2 = c_host_cublas[i];
+ // if (v1 / v2 > 1.0001 || v2 / v1 > 1.0001 || abs(v1 - v2) > 1e-5) {
+ // errors++;
+ // if (errors < 10) printf("%f %f\n", v1, v2);
+ // }
+ //}
+
+ float wmmaTime;
+ cudaErrCheck(cudaEventElapsedTime(&wmmaTime, startWMMA, stopWMMA));
+ printf("wmma took %fms\n", wmmaTime);
+
+ cudaErrCheck(cudaEventDestroy(startWMMA));
+ cudaErrCheck(cudaEventDestroy(stopWMMA));
+ cudaErrCheck(cudaFree(a_fp32));
+ cudaErrCheck(cudaFree(b_fp32));
+ cudaErrCheck(cudaFree(a_fp16));
+ cudaErrCheck(cudaFree(b_fp16));
+ cudaErrCheck(cudaFree(c));
+ cudaErrCheck(cudaFree(c_wmma));
+ free(d_host_wmma);
+ free(c_host_wmma);
+ cudaErrCheck(cudaDeviceReset());
+ return 0;
+}
+
+
diff --git a/cuda-kernels/v4p_genericMatrixMultiply.cu b/cuda-kernels/v4p_genericMatrixMultiply.cu
new file mode 100644
index 0000000..a08903b
--- /dev/null
+++ b/cuda-kernels/v4p_genericMatrixMultiply.cu
@@ -0,0 +1,384 @@
+#include <stdio.h>
+#include <curand.h>
+
+// Define some error checking macros.
+#define cudaErrCheck(stat) { cudaErrCheck_((stat), __FILE__, __LINE__); }
+void cudaErrCheck_(cudaError_t stat, const char *file, int line) {
+ if (stat != cudaSuccess) {
+ fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(stat), file, line);
+ }
+}
+
+#define curandErrCheck(stat) { curandErrCheck_((stat), __FILE__, __LINE__); }
+void curandErrCheck_(curandStatus_t stat, const char *file, int line) {
+ if (stat != CURAND_STATUS_SUCCESS) {
+ fprintf(stderr, "cuRand Error: %d %s %d\n", stat, file, line);
+ }
+}
+
+#include <mma.h>
+using namespace nvcuda;
+
+// Must be multiples of 16 for wmma code to work
+#define MATRIX_M (32)
+#define MATRIX_N (32)
+#define MATRIX_K (32)
+
+
+// The only dimensions currently supported by WMMA
+const int WMMA_M = 16;
+const int WMMA_N = 16;
+const int WMMA_K = 16;
+
+
+
+__global__ void vp_example(int *a, int *b, int *c, int M, int N, int K ) {
+ // Leading dimensions. Packed with no transpositions.
+ int lda = M;
+ int ldb = K;
+ int ldc = M;
+
+ // Tile using a 2D grid
+ int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
+ int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
+
+ // Declare the fragments
+ int a_frag[8];
+ int b_frag[8];
+ int c_frag[8];
+ int acc_frag[8];
+
+ acc_frag[0]=0;
+ acc_frag[1]=0;
+ acc_frag[2]=0;
+ acc_frag[3]=0;
+ acc_frag[4]=0;
+ acc_frag[5]=0;
+ acc_frag[6]=0;
+ acc_frag[7]=0;
+
+ // Loop over k
+ for (int i = 0; i < K; i += WMMA_K) {
+ int aRow = warpM * WMMA_M;
+ int aCol = i;
+
+ int bRow = i;
+ int bCol = warpN * WMMA_N;
+
+ // Bounds checking
+ if (aRow < M && aCol < K && bRow < K && bCol < N) {
+ // Load the inputs
+ // vp::load_matrix_sync(a_frag, a + aRow * lda+ aCol , lda);
+ asm("/*");
+ asm("CPTX_BEGIN");
+ asm("vp.load.a.sync.row.m16n16k16.s32 {%0,%1,%2,%3,%4,%5,%6,%7},[%8],%9;" :
+ "=r"(a_frag[0]), "=r"(a_frag[1]),"=r"(a_frag[2]),"=r"(a_frag[3]),
+ "=r"(a_frag[4]),"=r"(a_frag[5]),"=r"(a_frag[6]),"=r"(a_frag[7]):
+ "l"(a+aRow*lda+aCol),"r"(lda)
+ );
+ asm("CPTX_END");
+ asm("*/");
+ //vp::load_matrix_sync(b_frag, b + bRow * ldb+ bCol , ldb);
+ asm("/*");
+ asm("CPTX_BEGIN");
+ asm("vp.load.b4.sync.row.m16n16k16.s32 {%0},[%1],%2;" :
+ "=r"(b_frag[0]):
+ "l"(b+bRow*ldb/8+bCol),"r"(ldb/8)
+ );
+ asm("CPTX_END");
+ asm("*/");
+
+ // Perform the matrix multiplication
+ //vp::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
+ asm("/*");
+ asm("CPTX_BEGIN");
+ asm("vp.mma.sync.row.row.m16n16k16.s32 {%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9, %10, %11, %12, %13, %14, %15}, {%16}, {%17, %18, %19, %20, %21, %22, %23, %24};" :
+ "=r"(acc_frag[0]), "=r"(acc_frag[1]),"=r"(acc_frag[2]),"=r"(acc_frag[3]),
+ "=r"(acc_frag[4]),"=r"(acc_frag[5]),"=r"(acc_frag[6]),"=r"(acc_frag[7]):
+ "r"(a_frag[0]),"r"(a_frag[1]),"r"(a_frag[2]),"r"(a_frag[3]),
+ "r"(a_frag[4]),"r"(a_frag[5]),"r"(a_frag[6]),"r"(a_frag[7]),
+ "r"(b_frag[0]),
+ "r"(acc_frag[0]),"r"(acc_frag[1]),"r"(acc_frag[2]),"r"(acc_frag[3]),
+ "r"(acc_frag[4]),"r"(acc_frag[5]),"r"(acc_frag[6]),"r"(acc_frag[7])
+ );
+ asm("CPTX_END");
+ asm("*/");
+
+ }
+ }
+
+ // Load in the current value of c, scale it by beta, and add this our result scaled by alpha
+ int cRow = warpM * WMMA_M;
+ int cCol = warpN * WMMA_N;
+
+ if (cRow < M && cCol < N) {
+ //vp::load_matrix_sync(c_frag, c + cRow*ldc + cCol , ldc, wmma::mem_row_major);
+ asm("/*");
+ asm("CPTX_BEGIN");
+ asm("vp.load.c.sync.row.m16n16k16.s32 {%0,%1,%2,%3,%4,%5,%6,%7},[%8],%9;" :
+ "=r"(c_frag[0]), "=r"(c_frag[1]),"=r"(c_frag[2]),"=r"(c_frag[3]),
+ "=r"(c_frag[4]),"=r"(c_frag[5]),"=r"(c_frag[6]),"=r"(c_frag[7]):
+ "l"(c+cRow*ldc),"r"(ldc)
+ );
+ asm("CPTX_END");
+ asm("*/");
+
+
+ for(int i=0; i < 8; i++) {
+ c_frag[i] = acc_frag[i] + c_frag[i];
+ }
+
+ // Store the output
+ //vp::store_matrix_sync(c + cRow *ldc + cCol , c_frag, ldc, wmma::mem_row_major);
+ asm("/*");
+ asm("CPTX_BEGIN");
+ asm("vp.store.d.sync.row.m16n16k16.s32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8},%9;" :
+ :"l"(c+cRow*ldc+cCol),
+ "r"(c_frag[0]), "r"(c_frag[1]),"r"(c_frag[2]),"r"(c_frag[3]),
+ "r"(c_frag[4]),"r"(c_frag[5]),"r"(c_frag[6]),"r"(c_frag[7]),
+ "r"(ldc)
+ );
+ asm("CPTX_END");
+ asm("*/");
+ }
+}
+
+__global__ void convertFp32ToFp16 (half *out, float *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n) {
+ out[idx] = in[idx];
+ }
+}
+__global__ void convertFp16ToFp32 (float *out, half *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n) {
+ out[idx] = in[idx];
+ }
+}
+
+__global__ void convertInt32ToInt4 (int *out, int *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n/8) {
+ out[idx] =(in[8*idx]&0xf)|(in[8*idx+1]&0xf)<<4|(in[8*idx+2]&0xf)<<8|(in[8*idx+3]&0xf)<<12|
+ (in[8*idx+4]&0xf)<<16|(in[8*idx+5]&0xf)<<20|(in[8*idx+6]&0xf)<<24|(in[8*idx+7]&0xf)<<28;
+ }
+}
+__global__ void convertInt32ToInt8 (int *out, int *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n/4) {
+ out[idx] =(in[4*idx]&0xff)|(in[4*idx+1]&0xff)<<8|(in[4*idx+2]&0xff)<<16|(in[4*idx+3]&0xff)<<24;
+ }
+}
+__global__ void convertInt32ToInt16 (int *out, int *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ if (idx < n/2) {
+ out[idx] =(in[2*idx]&0xffff)|(in[2*idx+1]&0xffff)<<16;
+ }
+}
+
+__global__ void convertInt4ToInt32 (int *out, int *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ int shft_amt=4*(idx%8);
+ int shft_mask=0xf<<shft_amt;
+ if (idx < n) {
+ out[idx]= (in[idx/8]&shft_mask)>>shft_amt;
+ }
+}
+__global__ void convertInt8ToInt32 (int *out, int *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ int shft_amt=8*(idx%4);
+ int shft_mask=0xff<<shft_amt;
+ if (idx < n) {
+ out[idx]= (in[idx/4]&shft_mask)>>shft_amt;
+ }
+}
+__global__ void convertInt16ToInt32 (int *out, int *in, int n) {
+ int idx = blockDim.x * blockIdx.x + threadIdx.x;
+ int shft_amt=16*(idx%2);
+ int shft_mask=0xffff<<shft_amt;
+ if (idx < n) {
+ out[idx]= (in[idx/2]&shft_mask)>>shft_amt;
+ }
+}
+
+int main(int argc, char* argv[]) {
+ int *a_int32;
+ int *b_int32;
+ int *c_int32;
+ int *d_int32;
+
+ int *a_int4;
+ int *b_int4;
+ int *a_int8;
+ int *b_int8;
+ int *a_int16;
+ int *b_int16;
+
+ int *a_host_wmma;
+ int *b_host_wmma;
+ int *c_host_wmma;
+ int *d_host_wmma;
+ int *d_cal_host_wmma;
+
+ cudaEvent_t startWMMA;
+ cudaEvent_t stopWMMA;
+
+
+ cudaErrCheck(cudaEventCreate(&startWMMA));
+ cudaErrCheck(cudaEventCreate(&stopWMMA));
+
+ // Use tensor cores
+ cudaErrCheck(cudaMalloc((void**)&a_int32, MATRIX_M * MATRIX_K * sizeof(int)));
+ cudaErrCheck(cudaMalloc((void**)&b_int32, MATRIX_K * MATRIX_N * sizeof(int)));
+ cudaErrCheck(cudaMalloc((void**)&c_int32, MATRIX_K * MATRIX_N * sizeof(int)));
+ cudaErrCheck(cudaMalloc((void**)&d_int32, MATRIX_K * MATRIX_N * sizeof(int)));
+ cudaErrCheck(cudaMalloc((void**)&a_int4, MATRIX_M * MATRIX_K * sizeof(int)/8));
+ cudaErrCheck(cudaMalloc((void**)&b_int4, MATRIX_K * MATRIX_N * sizeof(int)/8));
+ cudaErrCheck(cudaMalloc((void**)&a_int8, MATRIX_M * MATRIX_K * sizeof(int)/4));
+ cudaErrCheck(cudaMalloc((void**)&b_int8, MATRIX_K * MATRIX_N * sizeof(int)/4));
+ cudaErrCheck(cudaMalloc((void**)&a_int16, MATRIX_M * MATRIX_K * sizeof(int)/2));
+ cudaErrCheck(cudaMalloc((void**)&b_int16, MATRIX_K * MATRIX_N * sizeof(int)/2));
+
+
+ a_host_wmma = (int *)malloc(MATRIX_M * MATRIX_K * sizeof(int));
+ b_host_wmma = (int *)malloc(MATRIX_K * MATRIX_N * sizeof(int));
+ c_host_wmma = (int *)malloc(MATRIX_M * MATRIX_N * sizeof(int));
+ d_host_wmma = (int *)malloc(MATRIX_M * MATRIX_N * sizeof(int));
+ d_cal_host_wmma = (int *)malloc(MATRIX_M * MATRIX_N * sizeof(int));
+
+ printf("a_int32\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_K;n++){
+ a_host_wmma[m*MATRIX_K+n]=(m*MATRIX_K+n)%1024;
+ printf("%d ",a_host_wmma[m*MATRIX_K+n]);
+ }
+ printf(";\n");
+ }
+
+ printf("b_int32\n");
+ for(int m=0;m<MATRIX_K;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ b_host_wmma[m*MATRIX_N+n]=(m*MATRIX_N+n)%4;
+ printf("%d ",b_host_wmma[m*MATRIX_N+n]);
+ }
+ printf(";\n");
+ }
+
+ printf("c_int32\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ c_host_wmma[m*MATRIX_N+n]=(m*MATRIX_N+n)%4;
+ d_cal_host_wmma[m*MATRIX_N+n]=0;
+ printf("%d ",c_host_wmma[m*MATRIX_N+n]);
+ }
+ printf(";\n");
+ }
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ for(int k=0;k<MATRIX_K;k++){
+ d_cal_host_wmma[m*MATRIX_N+n]+= a_host_wmma[m*MATRIX_K+k]*b_host_wmma[k*MATRIX_K+n];
+ }
+ d_cal_host_wmma[m*MATRIX_N+n]+=c_host_wmma[m*MATRIX_N+n];
+ }
+ }
+
+
+ cudaErrCheck(cudaMemcpy(a_int32,a_host_wmma, MATRIX_M * MATRIX_K * sizeof(int), cudaMemcpyHostToDevice));
+ cudaErrCheck(cudaMemcpy(b_int32,b_host_wmma, MATRIX_K * MATRIX_N * sizeof(int), cudaMemcpyHostToDevice));
+ cudaErrCheck(cudaMemcpy(c_int32,c_host_wmma, MATRIX_M * MATRIX_N * sizeof(int), cudaMemcpyHostToDevice));
+
+ #ifdef TEST16
+ convertInt32ToInt16 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (a_int16, a_int32, MATRIX_M * MATRIX_K);
+ convertInt16ToInt32 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (d_int32, a_int16, MATRIX_M * MATRIX_K);
+ cudaErrCheck(cudaMemcpy(d_host_wmma, d_int32, MATRIX_M * MATRIX_N * sizeof(int), cudaMemcpyDeviceToHost));
+ #endif
+ #ifdef TEST8
+ convertInt32ToInt8 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (a_int8, a_int32, MATRIX_M * MATRIX_K);
+ convertInt8ToInt32 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (d_int32, a_int8, MATRIX_M * MATRIX_K);
+ cudaErrCheck(cudaMemcpy(d_host_wmma, d_int32, MATRIX_M * MATRIX_N * sizeof(int), cudaMemcpyDeviceToHost));
+ #endif
+ #ifdef TEST4
+ convertInt32ToInt4 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (b_int4, b_int32, MATRIX_M * MATRIX_K);
+ convertInt4ToInt32 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (d_int32, b_int4, MATRIX_M * MATRIX_K);
+ cudaErrCheck(cudaMemcpy(d_host_wmma, d_int32, MATRIX_M * MATRIX_N * sizeof(int), cudaMemcpyDeviceToHost));
+ #endif
+ convertInt32ToInt4 <<< (MATRIX_M * MATRIX_K + 255) / 256, 256 >>> (b_int4, b_int32, MATRIX_M * MATRIX_K);
+
+ dim3 gridDim;
+ dim3 blockDim;
+
+ // blockDim.x must be a multple of warpSize
+ // 128x4 means we have 16 warps and a block computes a 64x64 output tile
+ blockDim.x = 64;
+ blockDim.y = 2;
+
+ gridDim.x = (MATRIX_M + (WMMA_M * blockDim.x / 32 - 1)) / (WMMA_M * blockDim.x / 32);
+ gridDim.y = (MATRIX_N + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
+ printf("GRID:X=%d,Y=%d\n",gridDim.x,gridDim.y);
+ printf("BLOCK:X=%d,Y=%d\n",blockDim.x,blockDim.y);
+
+
+ printf("Running with wmma...\n");
+ cudaErrCheck(cudaEventRecord(startWMMA));
+ vp_example <<< gridDim, blockDim >>> (a_int32, b_int4, c_int32, MATRIX_M, MATRIX_N, MATRIX_K);
+ cudaErrCheck(cudaEventRecord(stopWMMA));
+ cudaErrCheck(cudaEventSynchronize(stopWMMA));
+
+ // Error checking
+ printf("\nChecking results...\n");
+ cudaErrCheck(cudaMemcpy(d_host_wmma, c_int32, MATRIX_M * MATRIX_N * sizeof(float), cudaMemcpyDeviceToHost));
+
+ float wmmaTime;
+ cudaErrCheck(cudaEventElapsedTime(&wmmaTime, startWMMA, stopWMMA));
+ printf("wmma took %fms\n", wmmaTime);
+
+ cudaErrCheck(cudaEventDestroy(startWMMA));
+ cudaErrCheck(cudaEventDestroy(stopWMMA));
+
+ int t=200000;
+ while(t-->0);
+ printf("D_CALCULATED\n");
+
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%d,",d_cal_host_wmma[m*MATRIX_N+n]);
+ }
+ printf("\n");
+ }
+ printf("D_WMMA\n");
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ printf("%d,",d_host_wmma[m*MATRIX_N+n]);
+ }
+ printf("\n");
+ }
+ int suc=1;
+ for(int m=0;m<MATRIX_M;m++){
+ for(int n=0;n<MATRIX_N;n++){
+ if(abs(d_cal_host_wmma[m*MATRIX_N+n]-d_host_wmma[m*MATRIX_N+n]))
+ {
+ printf("ERROR:\n");
+ suc=0;
+ }
+ }
+ }
+ if(suc==1)
+ printf("COMPLETED_SUCCESSFULLY\n");
+
+
+ cudaErrCheck(cudaFree(a_int32));
+ cudaErrCheck(cudaFree(b_int32));
+ cudaErrCheck(cudaFree(c_int32));
+ cudaErrCheck(cudaFree(d_int32));
+ cudaErrCheck(cudaFree(a_int8));
+ cudaErrCheck(cudaFree(b_int8));
+
+ free(a_host_wmma);
+ free(b_host_wmma);
+ free(c_host_wmma);
+ free(d_host_wmma);
+ cudaErrCheck(cudaDeviceReset());
+ return 0;
+}
+
+
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 1e84425..b26adfd 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -133,7 +133,7 @@ unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout
if(wmma_layout==ROW)
offset=load_c_float_row[thread_group]+16*in_tg_index;
else
- offset=load_c_float_col[thread_group]+16*in_tg_index;
+ offset=load_c_float_col[thread_group]+in_tg_index;
break;
default:
@@ -1810,7 +1810,7 @@ void vp_mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
printf("MATRIX_B\n");
for (i=0;i<16;i++){
for(j=0;j<16;j++){
- printf("%x ",matrix_b[i][j].s32);
+ printf("%d ",matrix_b[i][j].s32);
}
printf("\n");
}
@@ -1922,7 +1922,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
for(k=0;k<2*nelem;k++){
temp=nw_v[k].f16;
if(g_debug_instruction)
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
if(g_debug_instruction)
printf("\n");
@@ -1930,7 +1930,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
else{
if(g_debug_instruction){
for(k=0;k<8;k++){
- printf("%f ",v[k].f32);
+ printf("%.2f ",v[k].f32);
}
printf("\n");
}
@@ -1977,7 +1977,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
for (i=0;i<16;i++){
for(j=0;j<16;j++){
temp=matrix_a[i][j].f16;
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
printf("\n");
}
@@ -1985,7 +1985,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
for (i=0;i<16;i++){
for(j=0;j<16;j++){
temp=matrix_b[i][j].f16;
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
printf("\n");
}
@@ -1994,10 +1994,10 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
for(j=0;j<16;j++){
if(type2==F16_TYPE){
temp=matrix_c[i][j].f16;
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
else
- printf("%f ",matrix_c[i][j].f32);
+ printf("%.2f ",matrix_c[i][j].f32);
}
printf("\n");
}
@@ -2038,7 +2038,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
for(j=0;j<16;j++){
if(type==F16_TYPE){
temp=matrix_d[i][j].f16;
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
else
printf("%.2f ",matrix_d[i][j].f32);
@@ -2064,7 +2064,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
{
printf("thread%d:",thrd);
for(k=0;k<8;k++){
- printf("%f ",matrix_d[row_t[k]][col_t[k]].f32);
+ printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32);
}
printf("\n");
}
@@ -2074,7 +2074,7 @@ void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
printf("thread%d:",thrd);
for(k=0;k<8;k++){
temp=matrix_d[row_t[k]][col_t[k]].f16;
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
printf("\n");
@@ -3162,10 +3162,14 @@ void vp_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
addr_t new_addr = addr+thread_group_offset(thrd,VP_MMA,wmma_layout,type,stride)*size/8;
if(g_debug_instruction){
- printf("vp:store:thread%d=%d,%d,%d,%d,%d,%d,%d,%d\n",thrd,v[0].s32,v[1].s32,v[2].s32,v[3].s32,v[4].s32,v[5].s32,v[6].s32,v[7].s32);
+ printf("vp_st:thread%d=%d,%d,%d,%d,%d,%d,%d,%d\n",thrd,v[0].s32,v[1].s32,v[2].s32,v[3].s32,v[4].s32,v[5].s32,v[6].s32,v[7].s32);
}
+
for(k=0;k<8;k++){
- mem->write(new_addr+4*k,size/8,&v[k].s64,thread,pI);
+ if(wmma_layout==ROW)
+ mem->write(new_addr+4*k,size/8,&v[k].s64,thread,pI);
+ else if(wmma_layout==COL)
+ mem->write(new_addr+k*4*stride,size/8,&v[k].s64,thread,pI);
}
delete [] v;
@@ -3236,7 +3240,7 @@ void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
printf("thread=%d:",thrd);
for(l=0;l<8;l++){
temp=v[l].f32;
- printf("%f",temp);
+ printf("%.2f",temp);
}
printf("\n");
}
@@ -3292,6 +3296,7 @@ void vp_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst)
ptx_reg_t data[8];
addr_t new_addr;
+
//note we are using distribution of VP_MMA for every type of load!
if(wmma_type==LOAD_A||wmma_type==LOAD_C){
new_addr = addr+thread_group_offset(thrd,VP_MMA,wmma_layout,type,stride)*size/8;
@@ -3311,22 +3316,40 @@ void vp_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst)
if(wmma_type==LOAD_A||wmma_type==LOAD_C){
for(i=0;i<8;i++){
- mem->read(new_addr+4*i,size/8,&data[i].s64);
+ if(wmma_layout==ROW)
+ mem->read(new_addr+4*i,size/8,&data[i].s64);
+ else if(wmma_layout==COL)
+ mem->read(new_addr+4*stride*i,size/8,&data[i].s64);
}
}
else if(wmma_type==LOAD_B4){
- mem->read(new_addr,size/8,&data[0].s64);
+ if(wmma_layout==ROW){
+ mem->read(new_addr,size/8,&data[0].s64);
+ }
+ else if(wmma_layout==COL){
+
+ }
}
else if(wmma_type==LOAD_B8){
- mem->read(new_addr,size/8,&data[0].s64);
- mem->read(new_addr+4,size/8,&data[1].s64);
+ if(wmma_layout==ROW){
+ mem->read(new_addr,size/8,&data[0].s64);
+ mem->read(new_addr+4,size/8,&data[1].s64);
+ }
+ else if(wmma_layout==COL){
+
+ }
}
else if(wmma_type==LOAD_B16){
printf("LOADB16_MODE");
- mem->read(new_addr,size/8,&data[0].s64);
- mem->read(new_addr+4,size/8,&data[1].s64);
- mem->read(new_addr+8,size/8,&data[2].s64);
- mem->read(new_addr+12,size/8,&data[3].s64);
+ if(wmma_layout==ROW){
+ mem->read(new_addr,size/8,&data[0].s64);
+ mem->read(new_addr+4,size/8,&data[1].s64);
+ mem->read(new_addr+8,size/8,&data[2].s64);
+ mem->read(new_addr+12,size/8,&data[3].s64);
+ }
+ else if(wmma_layout==COL){
+
+ }
}
else{
printf("wrong vp_load type\n");;
@@ -3490,14 +3513,14 @@ void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
float temp;
for(i=0;i<16;i++){
temp=data[i].f16;
- printf("%f ",temp);
+ printf("%.2f ",temp);
}
printf("\n");
}
else{
printf("\nmma_ld:thread%d= ",thrd);
for(i=0;i<8;i++){
- printf("%f ",data[i].f32);
+ printf("%.2f ",data[i].f32);
}
printf("\n");
printf("\nmma_ld:thread%d= ",thrd);
generated by cgit v1.3 (git 2.53.0) at 2026-07-11 02:44:15 +0000