109 lines
16 KiB
Plaintext
109 lines
16 KiB
Plaintext
job 18003870 starting on gpub055.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.431740
|
|
-39.836346, -39.836452
|
|
-43.065910, -43.066021
|
|
-42.828045, -42.828144
|
|
-43.529541, -43.529644
|
|
-44.318398, -44.318497
|
|
-41.227425, -41.227528
|
|
-41.270760, -41.270863
|
|
-42.541393, -42.541523
|
|
-42.394997, -42.395119
|
|
---------------- Logits OK! (max err 0.000043, RMSE 0.000002) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-357a.qdstrm'
|
|
|
|
[1/8] [0% ] report_op_13.nsys-rep
|
|
[1/8] [0% ] report_op_13.nsys-rep
|
|
[1/8] [====26% ] report_op_13.nsys-rep
|
|
[1/8] [==========49% ] report_op_13.nsys-rep
|
|
[1/8] [===========50% ] report_op_13.nsys-rep
|
|
[1/8] [===========51% ] report_op_13.nsys-rep
|
|
[1/8] [=======38% ] report_op_13.nsys-rep
|
|
[1/8] [===========51% ] report_op_13.nsys-rep
|
|
[1/8] [===========52% ] report_op_13.nsys-rep
|
|
[1/8] [===========53% ] report_op_13.nsys-rep
|
|
[1/8] [==============64% ] report_op_13.nsys-rep
|
|
[1/8] [======================90% ] report_op_13.nsys-rep
|
|
[1/8] [========================100%] report_op_13.nsys-rep
|
|
[1/8] [========================100%] report_op_13.nsys-rep
|
|
|
|
[2/8] [0% ] report_op_13.sqlite
|
|
[2/8] [1% ] report_op_13.sqlite
|
|
[2/8] [2% ] report_op_13.sqlite
|
|
[2/8] [3% ] report_op_13.sqlite
|
|
[2/8] [4% ] report_op_13.sqlite
|
|
[2/8] [5% ] report_op_13.sqlite
|
|
[2/8] [6% ] report_op_13.sqlite
|
|
[2/8] [7% ] report_op_13.sqlite
|
|
[2/8] [8% ] report_op_13.sqlite
|
|
[2/8] [9% ] report_op_13.sqlite
|
|
[2/8] [10% ] report_op_13.sqlite
|
|
[2/8] [11% ] report_op_13.sqlite
|
|
[2/8] [12% ] report_op_13.sqlite
|
|
[2/8] [13% ] report_op_13.sqlite
|
|
[2/8] [14% ] report_op_13.sqlite
|
|
[2/8] [=15% ] report_op_13.sqlite
|
|
[2/8] [=16% ] report_op_13.sqlite
|
|
[2/8] [=17% ] report_op_13.sqlite
|
|
[2/8] [==18% ] report_op_13.sqlite
|
|
[2/8] [==19% ] report_op_13.sqlite
|
|
[2/8] [==20% ] report_op_13.sqlite
|
|
[2/8] [==21% ] report_op_13.sqlite
|
|
[2/8] [===22% ] report_op_13.sqlite
|
|
[2/8] [===23% ] report_op_13.sqlite
|
|
[2/8] [===24% ] report_op_13.sqlite
|
|
[2/8] [====25% ] report_op_13.sqlite
|
|
[2/8] [====26% ] report_op_13.sqlite
|
|
[2/8] [====27% ] report_op_13.sqlite
|
|
[2/8] [====28% ] report_op_13.sqlite
|
|
[2/8] [=====29% ] report_op_13.sqlite
|
|
[2/8] [=====30% ] report_op_13.sqlite
|
|
[2/8] [=====31% ] report_op_13.sqlite
|
|
[2/8] [=====32% ] report_op_13.sqlite
|
|
[2/8] [======33% ] report_op_13.sqlite
|
|
[2/8] [======34% ] report_op_13.sqlite
|
|
[2/8] [======35% ] report_op_13.sqlite
|
|
[2/8] [=======36% ] report_op_13.sqlite
|
|
[2/8] [=======37% ] report_op_13.sqlite
|
|
[2/8] [=======38% ] report_op_13.sqlite
|
|
[2/8] [=======39% ] report_op_13.sqlite
|
|
[2/8] [========40% ] report_op_13.sqlite
|
|
[2/8] [========41% ] report_op_13.sqlite
|
|
[2/8] [========42% ] report_op_13.sqlite
|
|
[2/8] [=========43% ] report_op_13.sqlite
|
|
[2/8] [=========44% ] report_op_13.sqlite
|
|
[2/8] [=========45% ] report_op_13.sqlite
|
|
[2/8] [=========46% ] report_op_13.sqlite
|
|
[2/8] [==========47% ] report_op_13.sqlite
|
|
[2/8] [==========48% ] report_op_13.sqlite
|
|
[2/8] [==========49% ] report_op_13.sqlite
|
|
[2/8] [===========50% ] report_op_13.sqlite
|
|
[2/8] [===========51% ] report_op_13.sqlite
|
|
[2/8] [===========52% ] report_op_13.sqlite
|
|
[2/8] [===========53% ] report_op_13.sqlite
|
|
[2/8] [============54% ] report_op_13.sqlite
|
|
[2/8] [============55% ] report_op_13.sqlite
|
|
[2/8] [============56% ] report_op_13.sqlite
|
|
[2/8] [============57% ] report_op_13.sqlite
|
|
[2/8] [=============58% ] report_op_13.sqlite
|
|
[2/8] [=============59% ] report_op_13.sqlite
|
|
[2/8] [=============60% ] report_op_13.sqlite
|
|
[2/8] [==============61% ] report_op_13.sqlite
|
|
[2/8] [==============62% ] report_op_13.sqlite
|
|
[2/8] [==============63% ] report_op_13.sqlite
|
|
[2/8] [==============64% ] report_op_13.sqlite
|
|
[2/8] [===============65% ] report_op_13.sqlite
|
|
[2/8] [===============66% ] report_op_13.sqlite
|
|
[2/8] [===============67% ] report_op_13.sqlite
|
|
[2/8] [================68% ] report_op_13.sqlite
|
|
[2/8] [================69% ] report_op_13.sqlite
|
|
[2/8] [================70% ] report_op_13.sqlite
|
|
[2/8] [================71% ] report_op_13.sqlite
|
|
[2/8] [=================72% ] report_op_13.sqlite
|
|
[2/8] [=================73% ] report_op_13.sqlite
|
|
[2/8] [=================74% ] report_op_13.sqlite
|
|
[2/8] [==================75% ] report_op_13.sqlite
|
|
[2/8] [==================76% ] report_op_13.sqlite
|
|
[2/8] [==================77% ] report_op_13.sqlite
|
|
[2/8] [==================78% ] report_op_13.sqlite
|
|
[2/8] [===================79% ] report_op_13.sqlite
|
|
[2/8] [===================80% ] report_op_13.sqlite
|
|
[2/8] [===================81% ] report_op_13.sqlite
|
|
[2/8] [===================82% ] report_op_13.sqlite
|
|
[2/8] [====================83% ] report_op_13.sqlite
|
|
[2/8] [====================84% ] report_op_13.sqlite
|
|
[2/8] [====================85% ] report_op_13.sqlite
|
|
[2/8] [=====================86% ] report_op_13.sqlite
|
|
[2/8] [=====================87% ] report_op_13.sqlite
|
|
[2/8] [=====================88% ] report_op_13.sqlite
|
|
[2/8] [=====================89% ] report_op_13.sqlite
|
|
[2/8] [======================90% ] report_op_13.sqlite
|
|
[2/8] [======================91% ] report_op_13.sqlite
|
|
[2/8] [======================92% ] report_op_13.sqlite
|
|
[2/8] [=======================93% ] report_op_13.sqlite
|
|
[2/8] [=======================94% ] report_op_13.sqlite
|
|
[2/8] [=======================95% ] report_op_13.sqlite
|
|
[2/8] [=======================96% ] report_op_13.sqlite
|
|
[2/8] [========================97% ] report_op_13.sqlite
|
|
[2/8] [========================98% ] report_op_13.sqlite
|
|
[2/8] [========================99% ] report_op_13.sqlite
|
|
[2/8] [========================100%] report_op_13.sqlite
|
|
[2/8] [========================100%] report_op_13.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ----------------------
|
|
62.3 1192177975 22 54189908.0 60718799.0 1293 100135381 46848153.7 poll
|
|
27.8 532636664 6 88772777.3 616568.5 7834 482699408 193951324.3 fread
|
|
7.6 145642532 586 248536.7 21555.0 1032 28349154 1447180.4 ioctl
|
|
1.1 20477585 1 20477585.0 20477585.0 20477585 20477585 0.0 pthread_cond_wait
|
|
0.5 9619823 38 253153.2 6146.0 3567 8444540 1368023.7 fopen
|
|
0.4 7231291 1 7231291.0 7231291.0 7231291 7231291 0.0 pthread_cond_broadcast
|
|
0.1 2321011 25 92840.4 9037.0 7353 1601907 317206.9 mmap64
|
|
0.1 1381765 10 138176.5 29961.0 22552 501578 193339.9 write
|
|
0.0 665856 32 20808.0 2144.0 1353 313456 72473.1 fclose
|
|
0.0 574715 17 33806.8 4579.0 1844 383638 91786.3 mmap
|
|
0.0 347480 2 173740.0 173740.0 156984 190496 23696.6 pthread_create
|
|
0.0 313434 55 5698.8 5871.0 2334 9788 2009.1 open64
|
|
0.0 206186 1 206186.0 206186.0 206186 206186 0.0 fflush
|
|
0.0 77074 1 77074.0 77074.0 77074 77074 0.0 fgets
|
|
0.0 48280 8 6035.0 4824.0 2665 12333 3569.5 munmap
|
|
0.0 35765 6 5960.8 6016.0 3055 8936 1976.5 open
|
|
0.0 23155 3 7718.3 9318.0 3457 10380 3728.4 pipe2
|
|
0.0 17893 9 1988.1 1463.0 1012 3617 1022.1 read
|
|
0.0 15579 2 7789.5 7789.5 5791 9788 2826.3 socket
|
|
0.0 9318 1 9318.0 9318.0 9318 9318 0.0 connect
|
|
0.0 6342 1 6342.0 6342.0 6342 6342 0.0 fopen64
|
|
0.0 3918 2 1959.0 1959.0 1453 2465 715.6 fwrite
|
|
0.0 1623 1 1623.0 1623.0 1623 1623 0.0 bind
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ---------------------------------
|
|
86.7 292338879 3 97446293.0 36179585.0 18745 256140549 138617465.5 cudaMemcpy
|
|
10.4 34929986 12 2910832.2 8676.5 230 29248496 8331465.4 cudaFree
|
|
1.1 3690547 171 21582.1 2875.0 2735 3041560 232340.0 cudaLaunchKernel
|
|
0.7 2477256 1 2477256.0 2477256.0 2477256 2477256 0.0 cudaFreeHost
|
|
0.4 1441077 1 1441077.0 1441077.0 1441077 1441077 0.0 cudaGetDeviceProperties_v2_v12000
|
|
0.4 1293189 1 1293189.0 1293189.0 1293189 1293189 0.0 cudaMallocHost
|
|
0.2 708847 6 118141.2 115361.0 3406 282809 107158.9 cudaMalloc
|
|
0.1 176229 838 210.3 180.0 89 5641 213.4 cuGetProcAddress_v2
|
|
0.0 34836 1 34836.0 34836.0 34836 34836 0.0 cudaMemset
|
|
0.0 16491 4 4122.8 3136.5 1021 9197 3611.3 cudaDeviceSynchronize
|
|
0.0 11905 18 661.4 280.5 250 3005 821.3 cudaEventCreateWithFlags
|
|
0.0 7888 18 438.2 270.5 210 2675 575.7 cudaEventDestroy
|
|
0.0 5102 4 1275.5 1278.0 1233 1313 33.0 cuInit
|
|
0.0 1753 3 584.3 461.0 110 1182 546.5 cuModuleGetLoadingMode
|
|
0.0 993 2 496.5 496.5 331 662 234.1 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- --------- --------- -------- -------- ----------- ----------------------------------------------------------------------------------------------------
|
|
97.6 243490921 49 4969202.5 4539029.0 1227366 73795414 10139382.4 matmul_forward_kernel(float *, const float *, const float *, const float *, int, int, int, int)
|
|
1.5 3693489 25 147739.6 148096.0 144832 149601 1394.9 layernorm_forward_kernel(float *, float *, float *, const float *, const float *, const float *, in…
|
|
0.5 1145094 12 95424.5 95232.5 95073 96065 370.8 preattention_scores_kernel(const float *, const float *, int, int, int, int, float *)
|
|
0.1 334432 12 27869.3 27968.0 26880 28704 511.5 attention_v_kernel(float *, const float *, const float *, int, int, int, int)
|
|
0.1 227360 12 18946.7 18976.0 18720 19008 74.1 gelu_forward_kernel(float *, const float *, int)
|
|
0.1 207585 12 17298.8 17344.0 17120 17440 111.3 softmax_forward_kernel(float *, float, const float *, int, int)
|
|
0.1 145664 24 6069.3 6128.0 5664 6464 351.1 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.0 96320 12 8026.7 8016.0 7840 8224 115.6 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.0 80128 1 80128.0 80128.0 80128 80128 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.0 70530 12 5877.5 5888.0 5856 5920 20.8 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ----------------------------
|
|
82.8 36065319 2 18032659.5 18032659.5 832 36064487 25500855.0 [CUDA memcpy Host-to-Device]
|
|
16.3 7099265 1 7099265.0 7099265.0 7099265 7099265 0.0 [CUDA memcpy Device-to-Host]
|
|
0.9 407490 1 407490.0 407490.0 407490 407490 0.0 [CUDA memset]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ----------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.393 1 232.393 232.393 232.393 232.393 0.000 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
|
|
Generated:
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_13.nsys-rep
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_13.sqlite
|